nginx: src/core/ngx_slab.c annotate

annotate src/core/ngx_slab.c @ 4729:4ff2131dbce5 stable-1.2

Merge of r4691: changed default alignment to 16. This fixes alignment problems observerd on ARMs, and likely also needed for MIPSes. Unless we know alignment is not required just assume we need 16, which appears to be safe default for all architectures. See here for details: http://mailman.nginx.org/pipermail/nginx/2012-June/034139.html

author	Maxim Dounin <mdounin@mdounin.ru>
date	Mon, 02 Jul 2012 17:08:26 +0000
parents	d620f497c50f
children	0ed8088f43b4

rev	line source
441 da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	1
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	2 /*
444 42d11f017717 nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright Igor Sysoev <igor@sysoev.ru> parents: 441 diff changeset	3 * Copyright (C) Igor Sysoev
4412 d620f497c50f Copyright updated. Maxim Konovalov <maxim@nginx.com> parents: 4224 diff changeset	4 * Copyright (C) Nginx, Inc.
441 da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	5 */
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	6
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	7 #include <ngx_config.h>
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	8 #include <ngx_core.h>
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	9
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	10
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	11 #define NGX_SLAB_PAGE_MASK 3
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	12 #define NGX_SLAB_PAGE 0
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	13 #define NGX_SLAB_BIG 1
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	14 #define NGX_SLAB_EXACT 2
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	15 #define NGX_SLAB_SMALL 3
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	16
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	17 #if (NGX_PTR_SIZE == 4)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	18
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	19 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	20 #define NGX_SLAB_PAGE_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	21 #define NGX_SLAB_PAGE_START 0x80000000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	22
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	23 #define NGX_SLAB_SHIFT_MASK 0x0000000f
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	24 #define NGX_SLAB_MAP_MASK 0xffff0000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	25 #define NGX_SLAB_MAP_SHIFT 16
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	26
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	27 #define NGX_SLAB_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	28
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	29 #else /* (NGX_PTR_SIZE == 8) */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	30
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	31 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	33 #define NGX_SLAB_PAGE_START 0x8000000000000000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	34
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	37 #define NGX_SLAB_MAP_SHIFT 32
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	38
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	39 #define NGX_SLAB_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	40
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	41 #endif
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	42
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	43
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	44 #if (NGX_DEBUG_MALLOC)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	45
4224 4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values. Igor Sysoev <igor@sysoev.ru> parents: 4223 diff changeset	46 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	47
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	48 #else
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	49
4223 1b779cb69dc8 malloc() debugging on MacOSX. Igor Sysoev <igor@sysoev.ru> parents: 2711 diff changeset	50 #if (NGX_HAVE_DEBUG_MALLOC)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	51
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	52 #define ngx_slab_junk(p, size) \
4224 4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values. Igor Sysoev <igor@sysoev.ru> parents: 4223 diff changeset	53 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	54
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	55 #else
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	56
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	57 #define ngx_slab_junk(p, size)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	58
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	59 #endif
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	60
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	61 #endif
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	62
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	63 static ngx_slab_page_t ngx_slab_alloc_pages(ngx_slab_pool_t pool,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	64 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	65 static void ngx_slab_free_pages(ngx_slab_pool_t pool, ngx_slab_page_t page,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	66 ngx_uint_t pages);
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	67 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	68 char *text);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	69
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	70
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	71 static ngx_uint_t ngx_slab_max_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	72 static ngx_uint_t ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	73 static ngx_uint_t ngx_slab_exact_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	74
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	75
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	76 void
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	77 ngx_slab_init(ngx_slab_pool_t *pool)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	78 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	79 u_char *p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	80 size_t size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	81 ngx_int_t m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	82 ngx_uint_t i, n, pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	83 ngx_slab_page_t *slots;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	84
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	85 /* STUB */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	86 if (ngx_slab_max_size == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	87 ngx_slab_max_size = ngx_pagesize / 2;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	88 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	89 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	90 /* void */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	91 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	92 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	93 /**/
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	94
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	95 pool->min_size = 1 << pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	96
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	97 p = (u_char *) pool + sizeof(ngx_slab_pool_t);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	98 size = pool->end - p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	99
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	100 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	101
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	102 slots = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	103 n = ngx_pagesize_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	104
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	105 for (i = 0; i < n; i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	106 slots[i].slab = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	107 slots[i].next = &slots[i];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	108 slots[i].prev = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	109 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	110
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	111 p += n * sizeof(ngx_slab_page_t);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	112
1645 31622d9f2c0d the simple expression has the same precision without overflow Igor Sysoev <igor@sysoev.ru> parents: 1018 diff changeset	113 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	114
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	115 ngx_memzero(p, pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	116
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	117 pool->pages = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	118
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	119 pool->free.prev = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	120 pool->free.next = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	121
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	122 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	123 pool->pages->next = &pool->free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	124 pool->pages->prev = (uintptr_t) &pool->free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	125
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	126 pool->start = (u_char *)
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	127 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t),
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	128 ngx_pagesize);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	129
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	130 m = pages - (pool->end - pool->start) / ngx_pagesize;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	131 if (m > 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	132 pages -= m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	133 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	134 }
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	135
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	136 pool->log_ctx = &pool->zero;
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	137 pool->zero = '\0';
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	138 }
357 e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	139
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	140
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	141 void *
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	142 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	143 {
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	144 void *p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	145
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	146 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	147
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	148 p = ngx_slab_alloc_locked(pool, size);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	149
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	150 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	151
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	152 return p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	153 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	154
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	155
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	156 void *
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	157 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	158 {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	159 size_t s;
881 5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	160 uintptr_t p, n, m, mask, *bitmap;
5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	161 ngx_uint_t i, slot, shift, map;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	162 ngx_slab_page_t page, prev, *slots;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	163
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	164 if (size >= ngx_slab_max_size) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	165
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	166 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
967 1cb178720355 style fix Igor Sysoev <igor@sysoev.ru> parents: 966 diff changeset	167 "slab alloc: %uz", size);
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	168
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	169 page = ngx_slab_alloc_pages(pool, (size + ngx_pagesize - 1)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	170 >> ngx_pagesize_shift);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	171 if (page) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	172 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	173 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	174
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	175 } else {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	176 p = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	177 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	178
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	179 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	180 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	181
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	182 if (size > pool->min_size) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	183 shift = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	184 for (s = size - 1; s >>= 1; shift++) { /* void */ }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	185 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	186
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	187 } else {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	188 size = pool->min_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	189 shift = pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	190 slot = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	191 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	192
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	193 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	194 "slab alloc: %uz slot: %ui", size, slot);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	195
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	196 slots = (ngx_slab_page_t ) ((u_char ) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	197 page = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	198
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	199 if (page->next != page) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	200
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	201 if (shift < ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	202
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	203 do {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	204 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	205 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	206
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	207 map = (1 << (ngx_pagesize_shift - shift))
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	208 / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	209
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	210 for (n = 0; n < map; n++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	211
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	212 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	213
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	214 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	215 if ((bitmap[n] & m)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	216 continue;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	217 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	218
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	219 bitmap[n] \|= m;
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	220
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	221 i = ((n * sizeof(uintptr_t) * 8) << shift)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	222 + (i << shift);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	223
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	224 if (bitmap[n] == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	225 for (n = n + 1; n < map; n++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	226 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	227 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	228
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	229 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	230 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	231 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	232
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	233 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	234 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	235 prev->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	236 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	237
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	238 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	239 page->prev = NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	240 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	241
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	242 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	243
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	244 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	245 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	246 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	247 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	248
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	249 page = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	250
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	251 } while (page);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	252
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	253 } else if (shift == ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	254
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	255 do {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	256 if (page->slab != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	257
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	258 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	259 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	260 continue;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	261 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	262
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	263 page->slab \|= m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	264
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	265 if (page->slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	266 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	267 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	268 prev->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	269 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	270
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	271 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	272 page->prev = NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	273 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	274
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	275 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	276 p += i << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	277 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	278
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	279 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	280 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	281 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	282
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	283 page = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	284
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	285 } while (page);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	286
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	287 } else { /* shift > ngx_slab_exact_shift */
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	288
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	289 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	290 n = 1 << n;
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	291 n = ((uintptr_t) 1 << n) - 1;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	292 mask = n << NGX_SLAB_MAP_SHIFT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	293
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	294 do {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	295 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	296
881 5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	297 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	298 m & mask;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	299 m <<= 1, i++)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	300 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	301 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	302 continue;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	303 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	304
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	305 page->slab \|= m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	306
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	307 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	308 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	309 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	310 prev->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	311 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	312
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	313 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	314 page->prev = NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	315 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	316
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	317 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	318 p += i << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	319 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	320
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	321 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	322 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	323 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	324
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	325 page = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	326
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	327 } while (page);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	328 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	329 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	330
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	331 page = ngx_slab_alloc_pages(pool, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	332
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	333 if (page) {
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	334 if (shift < ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	335 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	336 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	337
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	338 s = 1 << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	339 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	340
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	341 if (n == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	342 n = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	343 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	344
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	345 bitmap[0] = (2 << n) - 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	346
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	347 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	348
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	349 for (i = 1; i < map; i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	350 bitmap[i] = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	351 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	352
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	353 page->slab = shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	354 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	355 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	356
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	357 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	358
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	359 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	360 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	361
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	362 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	363
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	364 } else if (shift == ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	365
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	366 page->slab = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	367 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	368 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	369
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	370 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	371
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	372 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	373 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	374
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	375 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	376
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	377 } else { /* shift > ngx_slab_exact_shift */
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	378
881 5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	379 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) \| shift;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	380 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	381 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	382
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	383 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	384
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	385 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	386 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	387
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	388 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	389 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	390 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	391
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	392 p = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	393
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	394 done:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	395
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	396 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab alloc: %p", p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	397
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	398 return (void *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	399 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	400
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	401
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	402 void
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	403 ngx_slab_free(ngx_slab_pool_t pool, void p)
357 e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	404 {
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	405 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	406
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	407 ngx_slab_free_locked(pool, p);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	408
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	409 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	410 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	411
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	412
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	413 void
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	414 ngx_slab_free_locked(ngx_slab_pool_t pool, void p)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	415 {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	416 size_t size;
1018 06306a20b8bb fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 1016 diff changeset	417 uintptr_t slab, m, *bitmap;
06306a20b8bb fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 1016 diff changeset	418 ngx_uint_t n, type, slot, shift, map;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	419 ngx_slab_page_t slots, page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	420
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	421 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	422
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	423 if ((u_char ) p < pool->start \|\| (u_char ) p > pool->end) {
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	424 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	425 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	426 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	427
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	428 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	429 page = &pool->pages[n];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	430 slab = page->slab;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	431 type = page->prev & NGX_SLAB_PAGE_MASK;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	432
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	433 switch (type) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	434
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	435 case NGX_SLAB_SMALL:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	436
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	437 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	438 size = 1 << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	439
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	440 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	441 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	442 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	443
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	444 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	445 m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1));
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	446 n /= (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	447 bitmap = (uintptr_t *) ((uintptr_t) p & ~(ngx_pagesize - 1));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	448
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	449 if (bitmap[n] & m) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	450
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	451 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	452 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	453 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	454 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	455
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	456 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	457 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	458
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	459 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	460 page->next->prev = (uintptr_t) page \| NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	461 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	462
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	463 bitmap[n] &= ~m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	464
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	465 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	466
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	467 if (n == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	468 n = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	469 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	470
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	471 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	472 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	473 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	474
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	475 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	476
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	477 for (n = 1; n < map; n++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	478 if (bitmap[n]) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	479 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	480 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	481 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	482
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	483 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	484
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	485 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	486 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	487
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	488 goto chunk_already_free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	489
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	490 case NGX_SLAB_EXACT:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	491
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	492 m = (uintptr_t) 1 <<
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	493 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	494 size = ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	495
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	496 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	497 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	498 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	499
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	500 if (slab & m) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	501 if (slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	502 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	503 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	504 slot = ngx_slab_exact_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	505
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	506 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	507 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	508
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	509 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	510 page->next->prev = (uintptr_t) page \| NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	511 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	512
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	513 page->slab &= ~m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	514
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	515 if (page->slab) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	516 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	517 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	518
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	519 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	520
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	521 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	522 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	523
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	524 goto chunk_already_free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	525
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	526 case NGX_SLAB_BIG:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	527
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	528 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	529 size = 1 << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	530
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	531 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	532 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	533 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	534
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	535 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	536 + NGX_SLAB_MAP_SHIFT);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	537
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	538 if (slab & m) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	539
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	540 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	541 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	542 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	543 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	544
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	545 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	546 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	547
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	548 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	549 page->next->prev = (uintptr_t) page \| NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	550 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	551
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	552 page->slab &= ~m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	553
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	554 if (page->slab & NGX_SLAB_MAP_MASK) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	555 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	556 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	557
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	558 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	559
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	560 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	561 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	562
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	563 goto chunk_already_free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	564
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	565 case NGX_SLAB_PAGE:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	566
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	567 if ((uintptr_t) p & (ngx_pagesize - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	568 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	569 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	570
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	571 if (slab == NGX_SLAB_PAGE_FREE) {
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	572 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	573 "ngx_slab_free(): page is already free");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	574 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	575 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	576
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	577 if (slab == NGX_SLAB_PAGE_BUSY) {
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	578 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	579 "ngx_slab_free(): pointer to wrong page");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	580 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	581 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	582
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	583 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	584 size = slab & ~NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	585
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	586 ngx_slab_free_pages(pool, &pool->pages[n], size);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	587
2711 6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC Igor Sysoev <igor@sysoev.ru> parents: 2611 diff changeset	588 ngx_slab_junk(p, size << ngx_pagesize_shift);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	589
2711 6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC Igor Sysoev <igor@sysoev.ru> parents: 2611 diff changeset	590 return;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	591 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	592
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	593 /* not reached */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	594
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	595 return;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	596
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	597 done:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	598
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	599 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	600
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	601 return;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	602
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	603 wrong_chunk:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	604
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	605 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	606 "ngx_slab_free(): pointer to wrong chunk");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	607
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	608 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	609
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	610 chunk_already_free:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	611
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	612 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	613 "ngx_slab_free(): chunk is already free");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	614
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	615 fail:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	616
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	617 return;
357 e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	618 }
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	619
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	620
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	621 static ngx_slab_page_t *
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	622 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	623 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	624 ngx_slab_page_t page, p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	625
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	626 for (page = pool->free.next; page != &pool->free; page = page->next) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	627
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	628 if (page->slab >= pages) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	629
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	630 if (page->slab > pages) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	631 page[pages].slab = page->slab - pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	632 page[pages].next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	633 page[pages].prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	634
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	635 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	636 p->next = &page[pages];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	637 page->next->prev = (uintptr_t) &page[pages];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	638
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	639 } else {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	640 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	641 p->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	642 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	643 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	644
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	645 page->slab = pages \| NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	646 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	647 page->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	648
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	649 if (--pages == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	650 return page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	651 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	652
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	653 for (p = page + 1; pages; pages--) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	654 p->slab = NGX_SLAB_PAGE_BUSY;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	655 p->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	656 p->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	657 p++;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	658 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	659
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	660 return page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	661 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	662 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	663
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	664 ngx_slab_error(pool, NGX_LOG_CRIT, "ngx_slab_alloc() failed: no memory");
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	665
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	666 return NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	667 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	668
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	669
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	670 static void
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	671 ngx_slab_free_pages(ngx_slab_pool_t pool, ngx_slab_page_t page,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	672 ngx_uint_t pages)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	673 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	674 ngx_slab_page_t *prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	675
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	676 page->slab = pages--;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	677
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	678 if (pages) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	679 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	680 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	681
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	682 if (page->next) {
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	683 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	684 prev->next = page->next;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	685 page->next->prev = page->prev;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	686 }
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	687
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	688 page->prev = (uintptr_t) &pool->free;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	689 page->next = pool->free.next;
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	690
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	691 page->next->prev = (uintptr_t) page;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	692
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	693 pool->free.next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	694 }
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	695
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	696
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	697 static void
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	698 ngx_slab_error(ngx_slab_pool_t pool, ngx_uint_t level, char text)
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	699 {
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	700 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	701 }

Mercurial > hg > nginx

annotate src/core/ngx_slab.c @ 4729:4ff2131dbce5 stable-1.2