Mercurial > hg > nginx

annotate src/core/ngx_slab.c @ 6825:f6beb55792de

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Slab: fixed the number of pages calculation. When estimating the number of pages, do not count memory for slots. In some cases this gives one extra usable memory page.
author Ruslan Ermilov <ru@nginx.com>
date Wed, 07 Dec 2016 22:25:37 +0300
parents ea12328518dc
children d0404c9a7675
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
1
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
2 /*
444
42d11f017717 nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents: 441
diff changeset
3 * Copyright (C) Igor Sysoev
4412
d620f497c50f Copyright updated.
Maxim Konovalov <maxim@nginx.com>
parents: 4224
diff changeset
4 * Copyright (C) Nginx, Inc.
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
5 */
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
6
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
7 #include <ngx_config.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
8 #include <ngx_core.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
9
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
10
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
11 #define NGX_SLAB_PAGE_MASK 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
12 #define NGX_SLAB_PAGE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
13 #define NGX_SLAB_BIG 1
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
14 #define NGX_SLAB_EXACT 2
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
15 #define NGX_SLAB_SMALL 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
17 #if (NGX_PTR_SIZE == 4)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
18
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
19 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
20 #define NGX_SLAB_PAGE_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
21 #define NGX_SLAB_PAGE_START 0x80000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
22
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
23 #define NGX_SLAB_SHIFT_MASK 0x0000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
24 #define NGX_SLAB_MAP_MASK 0xffff0000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
25 #define NGX_SLAB_MAP_SHIFT 16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
26
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
27 #define NGX_SLAB_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
28
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
29 #else /* (NGX_PTR_SIZE == 8) */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
30
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
31 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
33 #define NGX_SLAB_PAGE_START 0x8000000000000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
34
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
37 #define NGX_SLAB_MAP_SHIFT 32
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
38
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
39 #define NGX_SLAB_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
40
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
41 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
42
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
43
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
44 #define ngx_slab_slots(pool) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
45 (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t))
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
46
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
47 #define ngx_slab_page_type(page) ((page)->prev & NGX_SLAB_PAGE_MASK)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
48
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
49 #define ngx_slab_page_prev(page) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
50 (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
51
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
52 #define ngx_slab_page_addr(pool, page) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
53 ((((page) - (pool)->pages) << ngx_pagesize_shift) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
54 + (uintptr_t) (pool)->start)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
55
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
56
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
57 #if (NGX_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
58
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
59 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
60
4764
0ed8088f43b4 Reduced the number of preprocessor directives.
Ruslan Ermilov <ru@nginx.com>
parents: 4412
diff changeset
61 #elif (NGX_HAVE_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
62
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
63 #define ngx_slab_junk(p, size) \
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
64 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
65
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
66 #else
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
67
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
68 #define ngx_slab_junk(p, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
69
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
70 #endif
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
71
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
72 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
73 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
74 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
75 ngx_uint_t pages);
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
76 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
77 char *text);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
78
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
79
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
80 static ngx_uint_t ngx_slab_max_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
81 static ngx_uint_t ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
82 static ngx_uint_t ngx_slab_exact_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
83
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
84
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
85 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
86 ngx_slab_init(ngx_slab_pool_t *pool)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
87 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
88 u_char *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
89 size_t size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
90 ngx_int_t m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
91 ngx_uint_t i, n, pages;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
92 ngx_slab_page_t *slots, *page;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
93
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
94 /* STUB */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
95 if (ngx_slab_max_size == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
96 ngx_slab_max_size = ngx_pagesize / 2;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
97 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
98 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
99 /* void */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
100 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
101 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
102 /**/
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
103
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
104 pool->min_size = 1 << pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
105
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
106 slots = ngx_slab_slots(pool);
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
107
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
108 p = (u_char *) slots;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
109 size = pool->end - p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
110
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
111 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
112
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
113 n = ngx_pagesize_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
114
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
115 for (i = 0; i < n; i++) {
6824
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
116 /* only "next" is used in list head */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
117 slots[i].slab = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
118 slots[i].next = &slots[i];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
119 slots[i].prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
120 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
121
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
122 p += n * sizeof(ngx_slab_page_t);
6825
f6beb55792de Slab: fixed the number of pages calculation.
Ruslan Ermilov <ru@nginx.com>
parents: 6824
diff changeset
123 size -= n * sizeof(ngx_slab_page_t);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
124
1645
31622d9f2c0d the simple expression has the same precision without overflow
Igor Sysoev <igor@sysoev.ru>
parents: 1018
diff changeset
125 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
126
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
127 pool->pages = (ngx_slab_page_t *) p;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
128 ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
129
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
130 page = pool->pages;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
131
6824
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
132 /* only "next" is used in list head */
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
133 pool->free.slab = 0;
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
134 pool->free.next = page;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
135 pool->free.prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
136
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
137 page->slab = pages;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
138 page->next = &pool->free;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
139 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
140
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
141 pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t),
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
142 ngx_pagesize);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
143
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
144 m = pages - (pool->end - pool->start) / ngx_pagesize;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
145 if (m > 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
146 pages -= m;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
147 page->slab = pages;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
148 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
149
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
150 pool->last = pool->pages + pages;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
151
5634
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
152 pool->log_nomem = 1;
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
153 pool->log_ctx = &pool->zero;
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
154 pool->zero = '\0';
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
155 }
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
156
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
157
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
158 void *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
159 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
160 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
161 void *p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
162
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
163 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
164
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
165 p = ngx_slab_alloc_locked(pool, size);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
166
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
167 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
168
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
169 return p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
170 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
171
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
172
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
173 void *
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
174 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
175 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
176 size_t s;
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
177 uintptr_t p, n, m, mask, *bitmap;
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
178 ngx_uint_t i, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
179 ngx_slab_page_t *page, *prev, *slots;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
180
5766
abdb027be9d5 Core: bugfix for the ngx_slab_max_size case.
Jianjun Zheng <codeeply@gmail.com>
parents: 5726
diff changeset
181 if (size > ngx_slab_max_size) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
182
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
183 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
967
1cb178720355 style fix
Igor Sysoev <igor@sysoev.ru>
parents: 966
diff changeset
184 "slab alloc: %uz", size);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
185
4829
40de49cf6b37 Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents: 4764
diff changeset
186 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
40de49cf6b37 Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents: 4764
diff changeset
187 + ((size % ngx_pagesize) ? 1 : 0));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
188 if (page) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
189 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
190
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
191 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
192 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
193 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
194
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
195 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
196 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
197
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
198 if (size > pool->min_size) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
199 shift = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
200 for (s = size - 1; s >>= 1; shift++) { /* void */ }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
201 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
202
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
203 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
204 shift = pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
205 slot = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
206 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
207
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
208 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
209 "slab alloc: %uz slot: %ui", size, slot);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
210
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
211 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
212 page = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
213
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
214 if (page->next != page) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
215
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
216 if (shift < ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
217
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
218 do {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
219 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
220
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
221 map = (1 << (ngx_pagesize_shift - shift))
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
222 / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
223
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
224 for (n = 0; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
225
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
226 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
227
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
228 for (m = 1, i = 0; m; m <<= 1, i++) {
6807
ea76a3aa18ae Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6480
diff changeset
229 if (bitmap[n] & m) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
230 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
231 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
232
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
233 bitmap[n] |= m;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
234
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
235 i = ((n * sizeof(uintptr_t) * 8) << shift)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
236 + (i << shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
237
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
238 if (bitmap[n] == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
239 for (n = n + 1; n < map; n++) {
6474
Ruslan Ermilov <ru@nginx.com>
parents: 5766
diff changeset
240 if (bitmap[n] != NGX_SLAB_BUSY) {
Ruslan Ermilov <ru@nginx.com>
parents: 5766
diff changeset
241 p = (uintptr_t) bitmap + i;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
242
6474
Ruslan Ermilov <ru@nginx.com>
parents: 5766
diff changeset
243 goto done;
Ruslan Ermilov <ru@nginx.com>
parents: 5766
diff changeset
244 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
245 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
246
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
247 prev = ngx_slab_page_prev(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
248 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
249 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
250
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
251 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
252 page->prev = NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
253 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
254
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
255 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
256
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
257 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
258 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
259 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
260 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
261
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
262 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
263
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
264 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
265
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
266 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
267
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
268 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
269 if (page->slab != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
270
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
271 for (m = 1, i = 0; m; m <<= 1, i++) {
6807
ea76a3aa18ae Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6480
diff changeset
272 if (page->slab & m) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
273 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
274 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
275
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
276 page->slab |= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
277
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
278 if (page->slab == NGX_SLAB_BUSY) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
279 prev = ngx_slab_page_prev(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
280 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
281 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
282
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
283 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
284 page->prev = NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
285 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
286
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
287 p = ngx_slab_page_addr(pool, page) + (i << shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
288
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
289 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
290 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
291 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
292
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
293 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
294
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
295 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
296
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
297 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
298
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
299 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
300 n = 1 << n;
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
301 n = ((uintptr_t) 1 << n) - 1;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
302 mask = n << NGX_SLAB_MAP_SHIFT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
303
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
304 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
305 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
306
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
307 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
308 m & mask;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
309 m <<= 1, i++)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
310 {
6807
ea76a3aa18ae Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6480
diff changeset
311 if (page->slab & m) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
312 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
313 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
314
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
315 page->slab |= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
316
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
317 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
318 prev = ngx_slab_page_prev(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
319 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
320 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
321
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
322 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
323 page->prev = NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
324 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
325
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
326 p = ngx_slab_page_addr(pool, page) + (i << shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
327
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
328 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
329 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
330 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
331
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
332 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
333
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
334 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
335 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
336 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
337
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
338 page = ngx_slab_alloc_pages(pool, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
339
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
340 if (page) {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
341 if (shift < ngx_slab_exact_shift) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
342 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
343
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
344 s = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
345 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
346
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
347 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
348 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
349 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
350
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
351 bitmap[0] = (2 << n) - 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
352
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
353 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
354
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
355 for (i = 1; i < map; i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
356 bitmap[i] = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
357 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
358
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
359 page->slab = shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
360 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
361 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
362
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
363 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
364
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
365 p = ngx_slab_page_addr(pool, page) + s * n;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
366
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
367 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
368
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
369 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
370
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
371 page->slab = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
372 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
373 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
374
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
375 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
376
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
377 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
378
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
379 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
380
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
381 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
382
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
383 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
384 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
385 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
386
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
387 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
388
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
389 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
390
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
391 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
392 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
393 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
394
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
395 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
396
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
397 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
398
6480
f01ab2dbcfdc Fixed logging.
Sergey Kandaurov <pluknet@nginx.com>
parents: 6474
diff changeset
399 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
f01ab2dbcfdc Fixed logging.
Sergey Kandaurov <pluknet@nginx.com>
parents: 6474
diff changeset
400 "slab alloc: %p", (void *) p);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
401
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
402 return (void *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
403 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
404
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
405
5726
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
406 void *
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
407 ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size)
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
408 {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
409 void *p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
410
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
411 ngx_shmtx_lock(&pool->mutex);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
412
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
413 p = ngx_slab_calloc_locked(pool, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
414
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
415 ngx_shmtx_unlock(&pool->mutex);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
416
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
417 return p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
418 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
419
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
420
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
421 void *
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
422 ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size)
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
423 {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
424 void *p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
425
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
426 p = ngx_slab_alloc_locked(pool, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
427 if (p) {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
428 ngx_memzero(p, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
429 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
430
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
431 return p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
432 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
433
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
434
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
435 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
436 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
437 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
438 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
439
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
440 ngx_slab_free_locked(pool, p);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
441
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
442 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
443 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
444
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
445
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
446 void
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
447 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
448 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
449 size_t size;
1018
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
450 uintptr_t slab, m, *bitmap;
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
451 ngx_uint_t n, type, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
452 ngx_slab_page_t *slots, *page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
453
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
454 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
455
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
456 if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
457 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
458 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
459 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
460
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
461 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
462 page = &pool->pages[n];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
463 slab = page->slab;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
464 type = ngx_slab_page_type(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
465
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
466 switch (type) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
467
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
468 case NGX_SLAB_SMALL:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
469
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
470 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
471 size = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
472
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
473 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
474 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
475 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
476
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
477 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
478 m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
479 n /= (sizeof(uintptr_t) * 8);
5362
79b9101cecf4 Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents: 4829
diff changeset
480 bitmap = (uintptr_t *)
79b9101cecf4 Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents: 4829
diff changeset
481 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
482
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
483 if (bitmap[n] & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
484
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
485 if (page->next == NULL) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
486 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
487 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
488
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
489 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
490 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
491
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
492 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
493 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
494 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
495
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
496 bitmap[n] &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
497
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
498 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
499
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
500 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
501 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
502 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
503
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
504 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
505 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
506 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
507
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
508 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
509
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
510 for (n = 1; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
511 if (bitmap[n]) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
512 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
513 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
514 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
515
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
516 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
517
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
518 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
519 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
520
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
521 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
522
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
523 case NGX_SLAB_EXACT:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
524
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
525 m = (uintptr_t) 1 <<
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
526 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
527 size = ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
528
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
529 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
530 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
531 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
532
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
533 if (slab & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
534 if (slab == NGX_SLAB_BUSY) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
535 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
536 slot = ngx_slab_exact_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
537
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
538 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
539 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
540
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
541 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
542 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
543 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
544
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
545 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
546
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
547 if (page->slab) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
548 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
549 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
550
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
551 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
552
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
553 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
554 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
555
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
556 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
557
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
558 case NGX_SLAB_BIG:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
559
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
560 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
561 size = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
562
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
563 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
564 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
565 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
566
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
567 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
568 + NGX_SLAB_MAP_SHIFT);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
569
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
570 if (slab & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
571
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
572 if (page->next == NULL) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
573 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
574 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
575
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
576 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
577 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
578
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
579 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
580 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
581 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
582
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
583 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
584
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
585 if (page->slab & NGX_SLAB_MAP_MASK) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
586 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
587 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
588
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
589 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
590
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
591 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
592 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
593
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
594 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
595
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
596 case NGX_SLAB_PAGE:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
597
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
598 if ((uintptr_t) p & (ngx_pagesize - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
599 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
600 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
601
6809
87d7e640b45d Slab: improved double free detection.
Ruslan Ermilov <ru@nginx.com>
parents: 6808
diff changeset
602 if (!(slab & NGX_SLAB_PAGE_START)) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
603 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
604 "ngx_slab_free(): page is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
605 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
606 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
607
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
608 if (slab == NGX_SLAB_PAGE_BUSY) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
609 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
610 "ngx_slab_free(): pointer to wrong page");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
611 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
612 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
613
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
614 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
615 size = slab & ~NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
616
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
617 ngx_slab_free_pages(pool, &pool->pages[n], size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
618
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
619 ngx_slab_junk(p, size << ngx_pagesize_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
620
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
621 return;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
622 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
623
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
624 /* not reached */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
625
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
626 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
627
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
628 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
629
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
630 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
631
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
632 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
633
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
634 wrong_chunk:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
635
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
636 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
637 "ngx_slab_free(): pointer to wrong chunk");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
638
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
639 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
640
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
641 chunk_already_free:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
642
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
643 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
644 "ngx_slab_free(): chunk is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
645
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
646 fail:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
647
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
648 return;
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
649 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
650
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
651
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
652 static ngx_slab_page_t *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
653 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
654 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
655 ngx_slab_page_t *page, *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
656
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
657 for (page = pool->free.next; page != &pool->free; page = page->next) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
658
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
659 if (page->slab >= pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
660
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
661 if (page->slab > pages) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
662 page[page->slab - 1].prev = (uintptr_t) &page[pages];
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
663
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
664 page[pages].slab = page->slab - pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
665 page[pages].next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
666 page[pages].prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
667
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
668 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
669 p->next = &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
670 page->next->prev = (uintptr_t) &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
671
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
672 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
673 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
674 p->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
675 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
676 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
677
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
678 page->slab = pages | NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
679 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
680 page->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
681
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
682 if (--pages == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
683 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
684 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
685
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
686 for (p = page + 1; pages; pages--) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
687 p->slab = NGX_SLAB_PAGE_BUSY;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
688 p->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
689 p->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
690 p++;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
691 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
692
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
693 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
694 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
695 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
696
5634
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
697 if (pool->log_nomem) {
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
698 ngx_slab_error(pool, NGX_LOG_CRIT,
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
699 "ngx_slab_alloc() failed: no memory");
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
700 }
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
701
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
702 return NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
703 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
704
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
705
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
706 static void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
707 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
708 ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
709 {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
710 ngx_slab_page_t *prev, *join;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
711
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
712 page->slab = pages--;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
713
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
714 if (pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
715 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
716 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
717
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
718 if (page->next) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
719 prev = ngx_slab_page_prev(page);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
720 prev->next = page->next;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
721 page->next->prev = page->prev;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
722 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
723
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
724 join = page + page->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
725
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
726 if (join < pool->last) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
727
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
728 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
729
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
730 if (join->next != NULL) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
731 pages += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
732 page->slab += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
733
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
734 prev = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
735 prev->next = join->next;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
736 join->next->prev = join->prev;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
737
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
738 join->slab = NGX_SLAB_PAGE_FREE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
739 join->next = NULL;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
740 join->prev = NGX_SLAB_PAGE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
741 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
742 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
743 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
744
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
745 if (page > pool->pages) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
746 join = page - 1;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
747
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
748 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
749
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
750 if (join->slab == NGX_SLAB_PAGE_FREE) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
751 join = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
752 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
753
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
754 if (join->next != NULL) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
755 pages += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
756 join->slab += page->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
757
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
758 prev = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
759 prev->next = join->next;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
760 join->next->prev = join->prev;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
761
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
762 page->slab = NGX_SLAB_PAGE_FREE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
763 page->next = NULL;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
764 page->prev = NGX_SLAB_PAGE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
765
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
766 page = join;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
767 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
768 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
769 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
770
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
771 if (pages) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
772 page[pages].prev = (uintptr_t) page;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
773 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
774
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
775 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
776 page->next = pool->free.next;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
777
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
778 page->next->prev = (uintptr_t) page;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
779
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
780 pool->free.next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
781 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
782
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
783
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
784 static void
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
785 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text)
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
786 {
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
787 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
788 }