nginx: src/core/ngx_slab.c annotate

annotate src/core/ngx_slab.c @ 5036:6e374742043e stable-1.2

Merge of r4944: removed GLOB_NOSORT glob option. This will result in alphabetical sorting of included files if the "include" directive with wildcards is used. Note that the behaviour is now different from that on Windows, where alphabetical sorting is not guaranteed for FindFirsFile()/FindNextFile() (used to be alphabetical on NTFS, but not on FAT). Approved by Igor Sysoev, prodded by many.

author	Maxim Dounin <mdounin@mdounin.ru>
date	Sat, 09 Feb 2013 21:27:37 +0000
parents	a0f1d53546d4
children	79b9101cecf4

rev	line source
441 da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	1
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	2 /*
444 42d11f017717 nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright Igor Sysoev <igor@sysoev.ru> parents: 441 diff changeset	3 * Copyright (C) Igor Sysoev
4412 d620f497c50f Copyright updated. Maxim Konovalov <maxim@nginx.com> parents: 4224 diff changeset	4 * Copyright (C) Nginx, Inc.
441 da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	5 */
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files Igor Sysoev <igor@sysoev.ru> parents: 357 diff changeset	6
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	7 #include <ngx_config.h>
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	8 #include <ngx_core.h>
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	9
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	10
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	11 #define NGX_SLAB_PAGE_MASK 3
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	12 #define NGX_SLAB_PAGE 0
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	13 #define NGX_SLAB_BIG 1
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	14 #define NGX_SLAB_EXACT 2
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	15 #define NGX_SLAB_SMALL 3
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	16
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	17 #if (NGX_PTR_SIZE == 4)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	18
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	19 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	20 #define NGX_SLAB_PAGE_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	21 #define NGX_SLAB_PAGE_START 0x80000000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	22
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	23 #define NGX_SLAB_SHIFT_MASK 0x0000000f
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	24 #define NGX_SLAB_MAP_MASK 0xffff0000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	25 #define NGX_SLAB_MAP_SHIFT 16
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	26
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	27 #define NGX_SLAB_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	28
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	29 #else /* (NGX_PTR_SIZE == 8) */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	30
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	31 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	33 #define NGX_SLAB_PAGE_START 0x8000000000000000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	34
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	37 #define NGX_SLAB_MAP_SHIFT 32
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	38
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	39 #define NGX_SLAB_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	40
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	41 #endif
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	42
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	43
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	44 #if (NGX_DEBUG_MALLOC)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	45
4224 4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values. Igor Sysoev <igor@sysoev.ru> parents: 4223 diff changeset	46 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	47
4801 8f2f71233f5b Merge of r4765: reduced the number of preprocessor directives. Maxim Dounin <mdounin@mdounin.ru> parents: 4412 diff changeset	48 #elif (NGX_HAVE_DEBUG_MALLOC)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	49
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	50 #define ngx_slab_junk(p, size) \
4224 4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values. Igor Sysoev <igor@sysoev.ru> parents: 4223 diff changeset	51 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	52
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	53 #else
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	54
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	55 #define ngx_slab_junk(p, size)
1771 b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	56
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	57 #endif
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator Igor Sysoev <igor@sysoev.ru> parents: 1645 diff changeset	58
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	59 static ngx_slab_page_t ngx_slab_alloc_pages(ngx_slab_pool_t pool,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	60 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	61 static void ngx_slab_free_pages(ngx_slab_pool_t pool, ngx_slab_page_t page,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	62 ngx_uint_t pages);
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	63 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	64 char *text);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	65
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	66
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	67 static ngx_uint_t ngx_slab_max_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	68 static ngx_uint_t ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	69 static ngx_uint_t ngx_slab_exact_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	70
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	71
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	72 void
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	73 ngx_slab_init(ngx_slab_pool_t *pool)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	74 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	75 u_char *p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	76 size_t size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	77 ngx_int_t m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	78 ngx_uint_t i, n, pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	79 ngx_slab_page_t *slots;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	80
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	81 /* STUB */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	82 if (ngx_slab_max_size == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	83 ngx_slab_max_size = ngx_pagesize / 2;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	84 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	85 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	86 /* void */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	87 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	88 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	89 /**/
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	90
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	91 pool->min_size = 1 << pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	92
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	93 p = (u_char *) pool + sizeof(ngx_slab_pool_t);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	94 size = pool->end - p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	95
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	96 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	97
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	98 slots = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	99 n = ngx_pagesize_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	100
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	101 for (i = 0; i < n; i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	102 slots[i].slab = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	103 slots[i].next = &slots[i];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	104 slots[i].prev = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	105 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	106
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	107 p += n * sizeof(ngx_slab_page_t);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	108
1645 31622d9f2c0d the simple expression has the same precision without overflow Igor Sysoev <igor@sysoev.ru> parents: 1018 diff changeset	109 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	110
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	111 ngx_memzero(p, pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	112
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	113 pool->pages = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	114
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	115 pool->free.prev = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	116 pool->free.next = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	117
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	118 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	119 pool->pages->next = &pool->free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	120 pool->pages->prev = (uintptr_t) &pool->free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	121
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	122 pool->start = (u_char *)
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	123 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t),
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	124 ngx_pagesize);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	125
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	126 m = pages - (pool->end - pool->start) / ngx_pagesize;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	127 if (m > 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	128 pages -= m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	129 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	130 }
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	131
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	132 pool->log_ctx = &pool->zero;
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	133 pool->zero = '\0';
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	134 }
357 e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	135
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	136
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	137 void *
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	138 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	139 {
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	140 void *p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	141
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	142 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	143
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	144 p = ngx_slab_alloc_locked(pool, size);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	145
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	146 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	147
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	148 return p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	149 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	150
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	151
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	152 void *
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	153 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	154 {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	155 size_t s;
881 5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	156 uintptr_t p, n, m, mask, *bitmap;
5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	157 ngx_uint_t i, slot, shift, map;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	158 ngx_slab_page_t page, prev, *slots;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	159
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	160 if (size >= ngx_slab_max_size) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	161
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	162 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
967 1cb178720355 style fix Igor Sysoev <igor@sysoev.ru> parents: 966 diff changeset	163 "slab alloc: %uz", size);
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	164
4848 a0f1d53546d4 Merge of r4778, r4782, r4783, r4824, r4830, r4834: minor fixes. Maxim Dounin <mdounin@mdounin.ru> parents: 4801 diff changeset	165 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
a0f1d53546d4 Merge of r4778, r4782, r4783, r4824, r4830, r4834: minor fixes. Maxim Dounin <mdounin@mdounin.ru> parents: 4801 diff changeset	166 + ((size % ngx_pagesize) ? 1 : 0));
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	167 if (page) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	168 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	169 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	170
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	171 } else {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	172 p = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	173 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	174
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	175 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	176 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	177
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	178 if (size > pool->min_size) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	179 shift = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	180 for (s = size - 1; s >>= 1; shift++) { /* void */ }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	181 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	182
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	183 } else {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	184 size = pool->min_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	185 shift = pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	186 slot = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	187 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	188
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	189 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	190 "slab alloc: %uz slot: %ui", size, slot);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	191
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	192 slots = (ngx_slab_page_t ) ((u_char ) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	193 page = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	194
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	195 if (page->next != page) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	196
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	197 if (shift < ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	198
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	199 do {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	200 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	201 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	202
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	203 map = (1 << (ngx_pagesize_shift - shift))
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	204 / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	205
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	206 for (n = 0; n < map; n++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	207
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	208 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	209
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	210 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	211 if ((bitmap[n] & m)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	212 continue;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	213 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	214
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	215 bitmap[n] \|= m;
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	216
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	217 i = ((n * sizeof(uintptr_t) * 8) << shift)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	218 + (i << shift);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	219
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	220 if (bitmap[n] == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	221 for (n = n + 1; n < map; n++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	222 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	223 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	224
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	225 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	226 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	227 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	228
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	229 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	230 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	231 prev->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	232 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	233
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	234 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	235 page->prev = NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	236 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	237
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	238 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	239
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	240 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	241 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	242 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	243 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	244
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	245 page = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	246
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	247 } while (page);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	248
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	249 } else if (shift == ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	250
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	251 do {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	252 if (page->slab != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	253
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	254 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	255 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	256 continue;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	257 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	258
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	259 page->slab \|= m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	260
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	261 if (page->slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	262 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	263 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	264 prev->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	265 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	266
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	267 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	268 page->prev = NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	269 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	270
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	271 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	272 p += i << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	273 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	274
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	275 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	276 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	277 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	278
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	279 page = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	280
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	281 } while (page);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	282
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	283 } else { /* shift > ngx_slab_exact_shift */
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	284
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	285 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	286 n = 1 << n;
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	287 n = ((uintptr_t) 1 << n) - 1;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	288 mask = n << NGX_SLAB_MAP_SHIFT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	289
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	290 do {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	291 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	292
881 5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	293 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	294 m & mask;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	295 m <<= 1, i++)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	296 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	297 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	298 continue;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	299 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	300
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	301 page->slab \|= m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	302
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	303 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	304 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	305 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	306 prev->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	307 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	308
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	309 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	310 page->prev = NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	311 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	312
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	313 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	314 p += i << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	315 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	316
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	317 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	318 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	319 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	320
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	321 page = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	322
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	323 } while (page);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	324 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	325 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	326
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	327 page = ngx_slab_alloc_pages(pool, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	328
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	329 if (page) {
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	330 if (shift < ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	331 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	332 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	333
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	334 s = 1 << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	335 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	336
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	337 if (n == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	338 n = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	339 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	340
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	341 bitmap[0] = (2 << n) - 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	342
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	343 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	344
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	345 for (i = 1; i < map; i++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	346 bitmap[i] = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	347 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	348
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	349 page->slab = shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	350 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	351 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	352
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	353 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	354
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	355 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	356 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	357
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	358 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	359
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	360 } else if (shift == ngx_slab_exact_shift) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	361
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	362 page->slab = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	363 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	364 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	365
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	366 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	367
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	368 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	369 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	370
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	371 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	372
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	373 } else { /* shift > ngx_slab_exact_shift */
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	374
881 5ed08a4c3bd3 fix building on 64-bit platforms Igor Sysoev <igor@sysoev.ru> parents: 860 diff changeset	375 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) \| shift;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	376 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	377 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	378
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	379 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	380
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	381 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	382 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	383
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	384 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	385 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	386 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	387
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	388 p = 0;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	389
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	390 done:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	391
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	392 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab alloc: %p", p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	393
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	394 return (void *) p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	395 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	396
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	397
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	398 void
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	399 ngx_slab_free(ngx_slab_pool_t pool, void p)
357 e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	400 {
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	401 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	402
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	403 ngx_slab_free_locked(pool, p);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	404
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	405 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	406 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	407
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	408
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	409 void
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	410 ngx_slab_free_locked(ngx_slab_pool_t pool, void p)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	411 {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	412 size_t size;
1018 06306a20b8bb fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 1016 diff changeset	413 uintptr_t slab, m, *bitmap;
06306a20b8bb fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 1016 diff changeset	414 ngx_uint_t n, type, slot, shift, map;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	415 ngx_slab_page_t slots, page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	416
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	417 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	418
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	419 if ((u_char ) p < pool->start \|\| (u_char ) p > pool->end) {
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	420 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	421 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	422 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	423
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	424 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	425 page = &pool->pages[n];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	426 slab = page->slab;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	427 type = page->prev & NGX_SLAB_PAGE_MASK;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	428
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	429 switch (type) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	430
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	431 case NGX_SLAB_SMALL:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	432
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	433 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	434 size = 1 << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	435
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	436 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	437 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	438 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	439
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	440 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	441 m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1));
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	442 n /= (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	443 bitmap = (uintptr_t *) ((uintptr_t) p & ~(ngx_pagesize - 1));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	444
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	445 if (bitmap[n] & m) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	446
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	447 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	448 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	449 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	450 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	451
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	452 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	453 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	454
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	455 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	456 page->next->prev = (uintptr_t) page \| NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	457 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	458
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	459 bitmap[n] &= ~m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	460
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	461 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	462
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	463 if (n == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	464 n = 1;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	465 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	466
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	467 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	468 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	469 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	470
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	471 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	472
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	473 for (n = 1; n < map; n++) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	474 if (bitmap[n]) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	475 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	476 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	477 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	478
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	479 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	480
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	481 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	482 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	483
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	484 goto chunk_already_free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	485
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	486 case NGX_SLAB_EXACT:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	487
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	488 m = (uintptr_t) 1 <<
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	489 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	490 size = ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	491
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	492 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	493 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	494 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	495
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	496 if (slab & m) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	497 if (slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	498 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	499 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	500 slot = ngx_slab_exact_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	501
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	502 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	503 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	504
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	505 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	506 page->next->prev = (uintptr_t) page \| NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	507 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	508
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	509 page->slab &= ~m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	510
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	511 if (page->slab) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	512 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	513 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	514
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	515 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	516
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	517 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	518 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	519
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	520 goto chunk_already_free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	521
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	522 case NGX_SLAB_BIG:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	523
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	524 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	525 size = 1 << shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	526
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	527 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	528 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	529 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	530
1016 d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	531 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms Igor Sysoev <igor@sysoev.ru> parents: 967 diff changeset	532 + NGX_SLAB_MAP_SHIFT);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	533
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	534 if (slab & m) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	535
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	536 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	537 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	538 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	539 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	540
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	541 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	542 slots[slot].next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	543
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	544 page->prev = (uintptr_t) &slots[slot] \| NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	545 page->next->prev = (uintptr_t) page \| NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	546 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	547
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	548 page->slab &= ~m;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	549
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	550 if (page->slab & NGX_SLAB_MAP_MASK) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	551 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	552 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	553
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	554 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	555
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	556 goto done;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	557 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	558
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	559 goto chunk_already_free;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	560
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	561 case NGX_SLAB_PAGE:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	562
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	563 if ((uintptr_t) p & (ngx_pagesize - 1)) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	564 goto wrong_chunk;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	565 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	566
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	567 if (slab == NGX_SLAB_PAGE_FREE) {
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	568 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	569 "ngx_slab_free(): page is already free");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	570 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	571 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	572
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	573 if (slab == NGX_SLAB_PAGE_BUSY) {
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	574 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	575 "ngx_slab_free(): pointer to wrong page");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	576 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	577 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	578
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	579 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	580 size = slab & ~NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	581
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	582 ngx_slab_free_pages(pool, &pool->pages[n], size);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	583
2711 6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC Igor Sysoev <igor@sysoev.ru> parents: 2611 diff changeset	584 ngx_slab_junk(p, size << ngx_pagesize_shift);
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	585
2711 6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC Igor Sysoev <igor@sysoev.ru> parents: 2611 diff changeset	586 return;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	587 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	588
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	589 /* not reached */
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	590
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	591 return;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	592
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	593 done:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	594
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	595 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	596
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	597 return;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	598
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	599 wrong_chunk:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	600
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	601 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	602 "ngx_slab_free(): pointer to wrong chunk");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	603
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	604 goto fail;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	605
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	606 chunk_already_free:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	607
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	608 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	609 "ngx_slab_free(): chunk is already free");
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	610
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	611 fail:
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	612
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	613 return;
357 e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import Igor Sysoev <igor@sysoev.ru> parents: diff changeset	614 }
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	615
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	616
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	617 static ngx_slab_page_t *
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	618 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	619 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	620 ngx_slab_page_t page, p;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	621
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	622 for (page = pool->free.next; page != &pool->free; page = page->next) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	623
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	624 if (page->slab >= pages) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	625
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	626 if (page->slab > pages) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	627 page[pages].slab = page->slab - pages;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	628 page[pages].next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	629 page[pages].prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	630
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	631 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	632 p->next = &page[pages];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	633 page->next->prev = (uintptr_t) &page[pages];
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	634
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	635 } else {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	636 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	637 p->next = page->next;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	638 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	639 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	640
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	641 page->slab = pages \| NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	642 page->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	643 page->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	644
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	645 if (--pages == 0) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	646 return page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	647 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	648
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	649 for (p = page + 1; pages; pages--) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	650 p->slab = NGX_SLAB_PAGE_BUSY;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	651 p->next = NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	652 p->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	653 p++;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	654 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	655
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	656 return page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	657 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	658 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	659
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	660 ngx_slab_error(pool, NGX_LOG_CRIT, "ngx_slab_alloc() failed: no memory");
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	661
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	662 return NULL;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	663 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	664
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	665
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	666 static void
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	667 ngx_slab_free_pages(ngx_slab_pool_t pool, ngx_slab_page_t page,
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	668 ngx_uint_t pages)
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	669 {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	670 ngx_slab_page_t *prev;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	671
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	672 page->slab = pages--;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	673
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	674 if (pages) {
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	675 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	676 }
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	677
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	678 if (page->next) {
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	679 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	680 prev->next = page->next;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	681 page->next->prev = page->prev;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	682 }
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	683
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	684 page->prev = (uintptr_t) &pool->free;
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	685 page->next = pool->free.next;
966 715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	686
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	687 page->next->prev = (uintptr_t) page;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked() Igor Sysoev <igor@sysoev.ru> parents: 881 diff changeset	688
860 201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	689 pool->free.next = page;
201d017ea470 slab allocator in shared memory Igor Sysoev <igor@sysoev.ru> parents: 826 diff changeset	690 }
2611 2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	691
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	692
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	693 static void
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	694 ngx_slab_error(ngx_slab_pool_t pool, ngx_uint_t level, char text)
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	695 {
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	696 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
2bce3f6416c6 improve ngx_slab_alloc() error logging Igor Sysoev <igor@sysoev.ru> parents: 2610 diff changeset	697 }

Mercurial > hg > nginx

annotate src/core/ngx_slab.c @ 5036:6e374742043e stable-1.2