Mercurial > hg > nginx

annotate src/core/ngx_slab.c @ 4412:d620f497c50f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Copyright updated.
author Maxim Konovalov <maxim@nginx.com>
date Wed, 18 Jan 2012 15:07:43 +0000
parents 4c4997de65b8
children 0ed8088f43b4
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
1
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
2 /*
444
42d11f017717 nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents: 441
diff changeset
3 * Copyright (C) Igor Sysoev
4412
d620f497c50f Copyright updated.
Maxim Konovalov <maxim@nginx.com>
parents: 4224
diff changeset
4 * Copyright (C) Nginx, Inc.
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
5 */
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
6
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
7 #include <ngx_config.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
8 #include <ngx_core.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
9
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
10
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
11 #define NGX_SLAB_PAGE_MASK 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
12 #define NGX_SLAB_PAGE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
13 #define NGX_SLAB_BIG 1
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
14 #define NGX_SLAB_EXACT 2
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
15 #define NGX_SLAB_SMALL 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
17 #if (NGX_PTR_SIZE == 4)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
18
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
19 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
20 #define NGX_SLAB_PAGE_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
21 #define NGX_SLAB_PAGE_START 0x80000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
22
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
23 #define NGX_SLAB_SHIFT_MASK 0x0000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
24 #define NGX_SLAB_MAP_MASK 0xffff0000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
25 #define NGX_SLAB_MAP_SHIFT 16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
26
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
27 #define NGX_SLAB_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
28
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
29 #else /* (NGX_PTR_SIZE == 8) */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
30
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
31 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
33 #define NGX_SLAB_PAGE_START 0x8000000000000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
34
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
37 #define NGX_SLAB_MAP_SHIFT 32
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
38
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
39 #define NGX_SLAB_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
40
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
41 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
42
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
43
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
44 #if (NGX_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
45
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
46 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
47
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
48 #else
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
49
4223
1b779cb69dc8 malloc() debugging on MacOSX.
Igor Sysoev <igor@sysoev.ru>
parents: 2711
diff changeset
50 #if (NGX_HAVE_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
51
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
52 #define ngx_slab_junk(p, size) \
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
53 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
54
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
55 #else
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
56
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
57 #define ngx_slab_junk(p, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
58
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
59 #endif
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
60
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
61 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
62
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
63 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
64 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
65 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
66 ngx_uint_t pages);
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
67 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
68 char *text);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
69
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
70
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
71 static ngx_uint_t ngx_slab_max_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
72 static ngx_uint_t ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
73 static ngx_uint_t ngx_slab_exact_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
74
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
75
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
76 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
77 ngx_slab_init(ngx_slab_pool_t *pool)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
78 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
79 u_char *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
80 size_t size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
81 ngx_int_t m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
82 ngx_uint_t i, n, pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
83 ngx_slab_page_t *slots;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
84
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
85 /* STUB */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
86 if (ngx_slab_max_size == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
87 ngx_slab_max_size = ngx_pagesize / 2;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
88 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
89 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
90 /* void */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
91 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
92 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
93 /**/
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
94
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
95 pool->min_size = 1 << pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
96
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
97 p = (u_char *) pool + sizeof(ngx_slab_pool_t);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
98 size = pool->end - p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
99
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
100 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
101
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
102 slots = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
103 n = ngx_pagesize_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
104
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
105 for (i = 0; i < n; i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
106 slots[i].slab = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
107 slots[i].next = &slots[i];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
108 slots[i].prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
109 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
110
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
111 p += n * sizeof(ngx_slab_page_t);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
112
1645
31622d9f2c0d the simple expression has the same precision without overflow
Igor Sysoev <igor@sysoev.ru>
parents: 1018
diff changeset
113 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
114
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
115 ngx_memzero(p, pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
116
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
117 pool->pages = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
118
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
119 pool->free.prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
120 pool->free.next = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
121
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
122 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
123 pool->pages->next = &pool->free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
124 pool->pages->prev = (uintptr_t) &pool->free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
125
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
126 pool->start = (u_char *)
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
127 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t),
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
128 ngx_pagesize);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
129
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
130 m = pages - (pool->end - pool->start) / ngx_pagesize;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
131 if (m > 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
132 pages -= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
133 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
134 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
135
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
136 pool->log_ctx = &pool->zero;
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
137 pool->zero = '\0';
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
138 }
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
139
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
140
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
141 void *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
142 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
143 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
144 void *p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
145
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
146 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
147
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
148 p = ngx_slab_alloc_locked(pool, size);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
149
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
150 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
151
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
152 return p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
153 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
154
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
155
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
156 void *
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
157 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
158 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
159 size_t s;
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
160 uintptr_t p, n, m, mask, *bitmap;
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
161 ngx_uint_t i, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
162 ngx_slab_page_t *page, *prev, *slots;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
163
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
164 if (size >= ngx_slab_max_size) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
165
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
166 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
967
1cb178720355 style fix
Igor Sysoev <igor@sysoev.ru>
parents: 966
diff changeset
167 "slab alloc: %uz", size);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
168
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
169 page = ngx_slab_alloc_pages(pool, (size + ngx_pagesize - 1)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
170 >> ngx_pagesize_shift);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
171 if (page) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
172 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
173 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
174
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
175 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
176 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
177 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
178
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
179 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
180 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
181
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
182 if (size > pool->min_size) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
183 shift = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
184 for (s = size - 1; s >>= 1; shift++) { /* void */ }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
185 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
186
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
187 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
188 size = pool->min_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
189 shift = pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
190 slot = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
191 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
192
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
193 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
194 "slab alloc: %uz slot: %ui", size, slot);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
195
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
196 slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
197 page = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
198
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
199 if (page->next != page) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
200
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
201 if (shift < ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
202
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
203 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
204 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
205 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
206
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
207 map = (1 << (ngx_pagesize_shift - shift))
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
208 / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
209
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
210 for (n = 0; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
211
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
212 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
213
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
214 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
215 if ((bitmap[n] & m)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
216 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
217 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
218
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
219 bitmap[n] |= m;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
220
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
221 i = ((n * sizeof(uintptr_t) * 8) << shift)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
222 + (i << shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
223
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
224 if (bitmap[n] == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
225 for (n = n + 1; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
226 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
227 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
228
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
229 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
230 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
231 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
232
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
233 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
234 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
235 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
236 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
237
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
238 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
239 page->prev = NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
240 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
241
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
242 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
243
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
244 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
245 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
246 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
247 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
248
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
249 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
250
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
251 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
252
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
253 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
254
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
255 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
256 if (page->slab != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
257
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
258 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
259 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
260 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
261 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
262
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
263 page->slab |= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
264
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
265 if (page->slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
266 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
267 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
268 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
269 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
270
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
271 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
272 page->prev = NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
273 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
274
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
275 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
276 p += i << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
277 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
278
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
279 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
280 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
281 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
282
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
283 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
284
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
285 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
286
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
287 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
288
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
289 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
290 n = 1 << n;
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
291 n = ((uintptr_t) 1 << n) - 1;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
292 mask = n << NGX_SLAB_MAP_SHIFT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
293
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
294 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
295 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
296
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
297 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
298 m & mask;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
299 m <<= 1, i++)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
300 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
301 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
302 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
303 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
304
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
305 page->slab |= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
306
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
307 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
308 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
309 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
310 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
311 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
312
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
313 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
314 page->prev = NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
315 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
316
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
317 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
318 p += i << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
319 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
320
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
321 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
322 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
323 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
324
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
325 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
326
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
327 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
328 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
329 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
330
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
331 page = ngx_slab_alloc_pages(pool, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
332
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
333 if (page) {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
334 if (shift < ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
335 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
336 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
337
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
338 s = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
339 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
340
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
341 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
342 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
343 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
344
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
345 bitmap[0] = (2 << n) - 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
346
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
347 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
348
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
349 for (i = 1; i < map; i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
350 bitmap[i] = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
351 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
352
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
353 page->slab = shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
354 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
355 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
356
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
357 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
358
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
359 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
360 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
361
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
362 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
363
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
364 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
365
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
366 page->slab = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
367 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
368 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
369
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
370 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
371
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
372 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
373 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
374
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
375 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
376
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
377 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
378
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
379 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
380 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
381 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
382
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
383 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
384
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
385 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
386 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
387
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
388 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
389 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
390 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
391
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
392 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
393
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
394 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
395
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
396 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab alloc: %p", p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
397
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
398 return (void *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
399 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
400
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
401
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
402 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
403 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
404 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
405 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
406
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
407 ngx_slab_free_locked(pool, p);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
408
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
409 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
410 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
411
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
412
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
413 void
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
414 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
415 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
416 size_t size;
1018
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
417 uintptr_t slab, m, *bitmap;
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
418 ngx_uint_t n, type, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
419 ngx_slab_page_t *slots, *page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
420
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
421 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
422
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
423 if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
424 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
425 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
426 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
427
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
428 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
429 page = &pool->pages[n];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
430 slab = page->slab;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
431 type = page->prev & NGX_SLAB_PAGE_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
432
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
433 switch (type) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
434
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
435 case NGX_SLAB_SMALL:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
436
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
437 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
438 size = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
439
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
440 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
441 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
442 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
443
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
444 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
445 m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
446 n /= (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
447 bitmap = (uintptr_t *) ((uintptr_t) p & ~(ngx_pagesize - 1));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
448
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
449 if (bitmap[n] & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
450
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
451 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
452 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
453 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
454 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
455
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
456 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
457 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
458
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
459 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
460 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
461 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
462
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
463 bitmap[n] &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
464
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
465 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
466
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
467 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
468 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
469 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
470
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
471 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
472 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
473 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
474
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
475 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
476
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
477 for (n = 1; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
478 if (bitmap[n]) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
479 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
480 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
481 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
482
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
483 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
484
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
485 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
486 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
487
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
488 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
489
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
490 case NGX_SLAB_EXACT:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
491
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
492 m = (uintptr_t) 1 <<
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
493 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
494 size = ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
495
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
496 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
497 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
498 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
499
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
500 if (slab & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
501 if (slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
502 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
503 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
504 slot = ngx_slab_exact_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
505
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
506 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
507 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
508
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
509 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
510 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
511 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
512
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
513 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
514
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
515 if (page->slab) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
516 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
517 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
518
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
519 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
520
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
521 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
522 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
523
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
524 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
525
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
526 case NGX_SLAB_BIG:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
527
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
528 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
529 size = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
530
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
531 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
532 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
533 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
534
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
535 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
536 + NGX_SLAB_MAP_SHIFT);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
537
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
538 if (slab & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
539
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
540 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
541 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
542 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
543 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
544
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
545 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
546 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
547
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
548 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
549 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
550 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
551
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
552 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
553
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
554 if (page->slab & NGX_SLAB_MAP_MASK) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
555 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
556 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
557
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
558 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
559
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
560 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
561 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
562
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
563 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
564
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
565 case NGX_SLAB_PAGE:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
566
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
567 if ((uintptr_t) p & (ngx_pagesize - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
568 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
569 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
570
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
571 if (slab == NGX_SLAB_PAGE_FREE) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
572 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
573 "ngx_slab_free(): page is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
574 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
575 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
576
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
577 if (slab == NGX_SLAB_PAGE_BUSY) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
578 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
579 "ngx_slab_free(): pointer to wrong page");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
580 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
581 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
582
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
583 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
584 size = slab & ~NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
585
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
586 ngx_slab_free_pages(pool, &pool->pages[n], size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
587
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
588 ngx_slab_junk(p, size << ngx_pagesize_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
589
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
590 return;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
591 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
592
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
593 /* not reached */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
594
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
595 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
596
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
597 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
598
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
599 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
600
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
601 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
602
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
603 wrong_chunk:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
604
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
605 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
606 "ngx_slab_free(): pointer to wrong chunk");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
607
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
608 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
609
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
610 chunk_already_free:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
611
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
612 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
613 "ngx_slab_free(): chunk is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
614
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
615 fail:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
616
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
617 return;
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
618 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
619
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
620
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
621 static ngx_slab_page_t *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
622 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
623 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
624 ngx_slab_page_t *page, *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
625
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
626 for (page = pool->free.next; page != &pool->free; page = page->next) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
627
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
628 if (page->slab >= pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
629
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
630 if (page->slab > pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
631 page[pages].slab = page->slab - pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
632 page[pages].next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
633 page[pages].prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
634
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
635 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
636 p->next = &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
637 page->next->prev = (uintptr_t) &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
638
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
639 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
640 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
641 p->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
642 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
643 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
644
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
645 page->slab = pages | NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
646 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
647 page->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
648
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
649 if (--pages == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
650 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
651 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
652
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
653 for (p = page + 1; pages; pages--) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
654 p->slab = NGX_SLAB_PAGE_BUSY;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
655 p->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
656 p->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
657 p++;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
658 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
659
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
660 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
661 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
662 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
663
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
664 ngx_slab_error(pool, NGX_LOG_CRIT, "ngx_slab_alloc() failed: no memory");
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
665
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
666 return NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
667 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
668
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
669
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
670 static void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
671 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
672 ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
673 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
674 ngx_slab_page_t *prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
675
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
676 page->slab = pages--;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
677
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
678 if (pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
679 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
680 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
681
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
682 if (page->next) {
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
683 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
684 prev->next = page->next;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
685 page->next->prev = page->prev;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
686 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
687
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
688 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
689 page->next = pool->free.next;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
690
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
691 page->next->prev = (uintptr_t) page;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
692
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
693 pool->free.next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
694 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
695
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
696
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
697 static void
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
698 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text)
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
699 {
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
700 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
701 }