Mercurial > hg > nginx

annotate src/core/ngx_slab.c @ 7050:8c5e3cc21332

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Slab: fixed small allocations on systems with large pagesize. Notably, on ppc64 with 64k pagesize, slab 0 (of size 8) requires 128 64-bit elements for bitmasks. The code bogusly assumed that one uintptr_t is enough for bitmasks plus at least one free slot.
author Ruslan Ermilov <ru@nginx.com>
date Tue, 04 Jul 2017 18:32:30 +0300
parents 94f30939545a
children 69f9ee0342db
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
1
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
2 /*
444
42d11f017717 nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents: 441
diff changeset
3 * Copyright (C) Igor Sysoev
4412
d620f497c50f Copyright updated.
Maxim Konovalov <maxim@nginx.com>
parents: 4224
diff changeset
4 * Copyright (C) Nginx, Inc.
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
5 */
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
6
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
7 #include <ngx_config.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
8 #include <ngx_core.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
9
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
10
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
11 #define NGX_SLAB_PAGE_MASK 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
12 #define NGX_SLAB_PAGE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
13 #define NGX_SLAB_BIG 1
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
14 #define NGX_SLAB_EXACT 2
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
15 #define NGX_SLAB_SMALL 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
17 #if (NGX_PTR_SIZE == 4)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
18
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
19 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
20 #define NGX_SLAB_PAGE_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
21 #define NGX_SLAB_PAGE_START 0x80000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
22
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
23 #define NGX_SLAB_SHIFT_MASK 0x0000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
24 #define NGX_SLAB_MAP_MASK 0xffff0000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
25 #define NGX_SLAB_MAP_SHIFT 16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
26
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
27 #define NGX_SLAB_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
28
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
29 #else /* (NGX_PTR_SIZE == 8) */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
30
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
31 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
33 #define NGX_SLAB_PAGE_START 0x8000000000000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
34
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
37 #define NGX_SLAB_MAP_SHIFT 32
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
38
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
39 #define NGX_SLAB_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
40
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
41 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
42
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
43
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
44 #define ngx_slab_slots(pool) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
45 (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t))
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
46
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
47 #define ngx_slab_page_type(page) ((page)->prev & NGX_SLAB_PAGE_MASK)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
48
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
49 #define ngx_slab_page_prev(page) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
50 (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
51
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
52 #define ngx_slab_page_addr(pool, page) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
53 ((((page) - (pool)->pages) << ngx_pagesize_shift) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
54 + (uintptr_t) (pool)->start)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
55
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
56
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
57 #if (NGX_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
58
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
59 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
60
4764
0ed8088f43b4 Reduced the number of preprocessor directives.
Ruslan Ermilov <ru@nginx.com>
parents: 4412
diff changeset
61 #elif (NGX_HAVE_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
62
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
63 #define ngx_slab_junk(p, size) \
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
64 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
65
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
66 #else
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
67
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
68 #define ngx_slab_junk(p, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
69
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
70 #endif
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
71
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
72 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
73 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
74 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
75 ngx_uint_t pages);
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
76 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
77 char *text);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
78
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
79
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
80 static ngx_uint_t ngx_slab_max_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
81 static ngx_uint_t ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
82 static ngx_uint_t ngx_slab_exact_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
83
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
84
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
85 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
86 ngx_slab_init(ngx_slab_pool_t *pool)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
87 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
88 u_char *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
89 size_t size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
90 ngx_int_t m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
91 ngx_uint_t i, n, pages;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
92 ngx_slab_page_t *slots, *page;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
93
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
94 /* STUB */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
95 if (ngx_slab_max_size == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
96 ngx_slab_max_size = ngx_pagesize / 2;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
97 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
98 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
99 /* void */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
100 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
101 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
102 /**/
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
103
6861
e4590dfd97ff Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents: 6830
diff changeset
104 pool->min_size = (size_t) 1 << pool->min_shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
105
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
106 slots = ngx_slab_slots(pool);
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
107
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
108 p = (u_char *) slots;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
109 size = pool->end - p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
110
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
111 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
112
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
113 n = ngx_pagesize_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
114
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
115 for (i = 0; i < n; i++) {
6824
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
116 /* only "next" is used in list head */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
117 slots[i].slab = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
118 slots[i].next = &slots[i];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
119 slots[i].prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
120 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
121
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
122 p += n * sizeof(ngx_slab_page_t);
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
123
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
124 pool->stats = (ngx_slab_stat_t *) p;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
125 ngx_memzero(pool->stats, n * sizeof(ngx_slab_stat_t));
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
126
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
127 p += n * sizeof(ngx_slab_stat_t);
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
128
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
129 size -= n * (sizeof(ngx_slab_page_t) + sizeof(ngx_slab_stat_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
130
1645
31622d9f2c0d the simple expression has the same precision without overflow
Igor Sysoev <igor@sysoev.ru>
parents: 1018
diff changeset
131 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
132
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
133 pool->pages = (ngx_slab_page_t *) p;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
134 ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
135
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
136 page = pool->pages;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
137
6824
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
138 /* only "next" is used in list head */
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
139 pool->free.slab = 0;
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
140 pool->free.next = page;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
141 pool->free.prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
142
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
143 page->slab = pages;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
144 page->next = &pool->free;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
145 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
146
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
147 pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t),
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
148 ngx_pagesize);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
149
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
150 m = pages - (pool->end - pool->start) / ngx_pagesize;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
151 if (m > 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
152 pages -= m;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
153 page->slab = pages;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
154 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
155
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
156 pool->last = pool->pages + pages;
6829
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
157 pool->pfree = pages;
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
158
5634
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
159 pool->log_nomem = 1;
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
160 pool->log_ctx = &pool->zero;
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
161 pool->zero = '\0';
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
162 }
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
163
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
164
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
165 void *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
166 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
167 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
168 void *p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
169
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
170 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
171
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
172 p = ngx_slab_alloc_locked(pool, size);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
173
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
174 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
175
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
176 return p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
177 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
178
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
179
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
180 void *
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
181 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
182 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
183 size_t s;
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
184 uintptr_t p, m, mask, *bitmap;
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
185 ngx_uint_t i, n, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
186 ngx_slab_page_t *page, *prev, *slots;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
187
5766
abdb027be9d5 Core: bugfix for the ngx_slab_max_size case.
Jianjun Zheng <codeeply@gmail.com>
parents: 5726
diff changeset
188 if (size > ngx_slab_max_size) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
189
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
190 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
967
1cb178720355 style fix
Igor Sysoev <igor@sysoev.ru>
parents: 966
diff changeset
191 "slab alloc: %uz", size);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
192
4829
40de49cf6b37 Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents: 4764
diff changeset
193 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
40de49cf6b37 Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents: 4764
diff changeset
194 + ((size % ngx_pagesize) ? 1 : 0));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
195 if (page) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
196 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
197
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
198 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
199 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
200 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
201
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
202 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
203 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
204
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
205 if (size > pool->min_size) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
206 shift = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
207 for (s = size - 1; s >>= 1; shift++) { /* void */ }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
208 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
209
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
210 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
211 shift = pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
212 slot = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
213 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
214
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
215 pool->stats[slot].reqs++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
216
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
217 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
218 "slab alloc: %uz slot: %ui", size, slot);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
219
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
220 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
221 page = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
222
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
223 if (page->next != page) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
224
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
225 if (shift < ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
226
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
227 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
228
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
229 map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
230
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
231 for (n = 0; n < map; n++) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
232
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
233 if (bitmap[n] != NGX_SLAB_BUSY) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
234
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
235 for (m = 1, i = 0; m; m <<= 1, i++) {
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
236 if (bitmap[n] & m) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
237 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
238 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
239
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
240 bitmap[n] |= m;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
241
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
242 i = (n * 8 * sizeof(uintptr_t) + i) << shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
243
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
244 p = (uintptr_t) bitmap + i;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
245
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
246 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
247
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
248 if (bitmap[n] == NGX_SLAB_BUSY) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
249 for (n = n + 1; n < map; n++) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
250 if (bitmap[n] != NGX_SLAB_BUSY) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
251 goto done;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
252 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
253 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
254
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
255 prev = ngx_slab_page_prev(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
256 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
257 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
258
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
259 page->next = NULL;
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
260 page->prev = NGX_SLAB_SMALL;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
261 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
262
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
263 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
264 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
265 }
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
266 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
267
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
268 } else if (shift == ngx_slab_exact_shift) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
269
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
270 for (m = 1, i = 0; m; m <<= 1, i++) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
271 if (page->slab & m) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
272 continue;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
273 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
274
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
275 page->slab |= m;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
276
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
277 if (page->slab == NGX_SLAB_BUSY) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
278 prev = ngx_slab_page_prev(page);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
279 prev->next = page->next;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
280 page->next->prev = page->prev;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
281
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
282 page->next = NULL;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
283 page->prev = NGX_SLAB_EXACT;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
284 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
285
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
286 p = ngx_slab_page_addr(pool, page) + (i << shift);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
287
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
288 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
289
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
290 goto done;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
291 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
292
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
293 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
294
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
295 mask = ((uintptr_t) 1 << (ngx_pagesize >> shift)) - 1;
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
296 mask <<= NGX_SLAB_MAP_SHIFT;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
297
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
298 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
299 m & mask;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
300 m <<= 1, i++)
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
301 {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
302 if (page->slab & m) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
303 continue;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
304 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
305
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
306 page->slab |= m;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
307
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
308 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
309 prev = ngx_slab_page_prev(page);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
310 prev->next = page->next;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
311 page->next->prev = page->prev;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
312
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
313 page->next = NULL;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
314 page->prev = NGX_SLAB_BIG;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
315 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
316
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
317 p = ngx_slab_page_addr(pool, page) + (i << shift);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
318
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
319 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
320
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
321 goto done;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
322 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
323 }
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
324
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
325 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_alloc(): page is busy");
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
326 ngx_debug_point();
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
327 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
328
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
329 page = ngx_slab_alloc_pages(pool, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
330
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
331 if (page) {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
332 if (shift < ngx_slab_exact_shift) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
333 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
334
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
335 n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
336
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
337 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
338 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
339 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
340
6830
6eed5ed31e22 Slab: commented bitmap initialization for small allocations.
Ruslan Ermilov <ru@nginx.com>
parents: 6829
diff changeset
341 /* "n" elements for bitmap, plus one requested */
7050
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
342
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
343 for (i = 0; i < (n + 1) / (8 * sizeof(uintptr_t)); i++) {
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
344 bitmap[i] = NGX_SLAB_BUSY;
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
345 }
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
346
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
347 m = ((uintptr_t) 1 << ((n + 1) % (8 * sizeof(uintptr_t)))) - 1;
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
348 bitmap[i] = m;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
349
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
350 map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
351
7050
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
352 for (i = i + 1; i < map; i++) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
353 bitmap[i] = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
354 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
355
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
356 page->slab = shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
357 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
358 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
359
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
360 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
361
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
362 pool->stats[slot].total += (ngx_pagesize >> shift) - n;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
363
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
364 p = ngx_slab_page_addr(pool, page) + (n << shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
365
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
366 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
367
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
368 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
369
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
370 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
371
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
372 page->slab = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
373 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
374 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
375
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
376 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
377
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
378 pool->stats[slot].total += 8 * sizeof(uintptr_t);
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
379
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
380 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
381
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
382 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
383
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
384 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
385
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
386 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
387
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
388 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
389 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
390 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
391
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
392 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
393
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
394 pool->stats[slot].total += ngx_pagesize >> shift;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
395
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
396 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
397
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
398 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
399
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
400 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
401 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
402 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
403
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
404 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
405
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
406 pool->stats[slot].fails++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
407
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
408 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
409
6480
f01ab2dbcfdc Fixed logging.
Sergey Kandaurov <pluknet@nginx.com>
parents: 6474
diff changeset
410 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
f01ab2dbcfdc Fixed logging.
Sergey Kandaurov <pluknet@nginx.com>
parents: 6474
diff changeset
411 "slab alloc: %p", (void *) p);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
412
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
413 return (void *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
414 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
415
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
416
5726
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
417 void *
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
418 ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size)
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
419 {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
420 void *p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
421
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
422 ngx_shmtx_lock(&pool->mutex);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
423
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
424 p = ngx_slab_calloc_locked(pool, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
425
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
426 ngx_shmtx_unlock(&pool->mutex);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
427
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
428 return p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
429 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
430
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
431
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
432 void *
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
433 ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size)
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
434 {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
435 void *p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
436
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
437 p = ngx_slab_alloc_locked(pool, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
438 if (p) {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
439 ngx_memzero(p, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
440 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
441
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
442 return p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
443 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
444
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
445
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
446 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
447 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
448 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
449 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
450
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
451 ngx_slab_free_locked(pool, p);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
452
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
453 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
454 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
455
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
456
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
457 void
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
458 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
459 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
460 size_t size;
1018
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
461 uintptr_t slab, m, *bitmap;
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
462 ngx_uint_t i, n, type, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
463 ngx_slab_page_t *slots, *page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
464
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
465 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
466
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
467 if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
468 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
469 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
470 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
471
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
472 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
473 page = &pool->pages[n];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
474 slab = page->slab;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
475 type = ngx_slab_page_type(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
476
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
477 switch (type) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
478
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
479 case NGX_SLAB_SMALL:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
480
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
481 shift = slab & NGX_SLAB_SHIFT_MASK;
6861
e4590dfd97ff Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents: 6830
diff changeset
482 size = (size_t) 1 << shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
483
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
484 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
485 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
486 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
487
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
488 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
489 m = (uintptr_t) 1 << (n % (8 * sizeof(uintptr_t)));
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
490 n /= 8 * sizeof(uintptr_t);
5362
79b9101cecf4 Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents: 4829
diff changeset
491 bitmap = (uintptr_t *)
79b9101cecf4 Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents: 4829
diff changeset
492 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
493
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
494 if (bitmap[n] & m) {
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
495 slot = shift - pool->min_shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
496
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
497 if (page->next == NULL) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
498 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
499
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
500 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
501 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
502
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
503 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
504 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
505 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
506
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
507 bitmap[n] &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
508
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
509 n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
510
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
511 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
512 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
513 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
514
7050
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
515 i = n / (8 * sizeof(uintptr_t));
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
516 m = ((uintptr_t) 1 << (n % (8 * sizeof(uintptr_t)))) - 1;
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
517
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
518 if (bitmap[i] & ~m) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
519 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
520 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
521
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
522 map = (ngx_pagesize >> shift) / (8 * sizeof(uintptr_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
523
7050
8c5e3cc21332 Slab: fixed small allocations on systems with large pagesize.
Ruslan Ermilov <ru@nginx.com>
parents: 7049
diff changeset
524 for (i = i + 1; i < map; i++) {
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
525 if (bitmap[i]) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
526 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
527 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
528 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
529
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
530 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
531
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
532 pool->stats[slot].total -= (ngx_pagesize >> shift) - n;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
533
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
534 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
535 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
536
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
537 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
538
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
539 case NGX_SLAB_EXACT:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
540
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
541 m = (uintptr_t) 1 <<
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
542 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
543 size = ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
544
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
545 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
546 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
547 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
548
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
549 if (slab & m) {
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
550 slot = ngx_slab_exact_shift - pool->min_shift;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
551
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
552 if (slab == NGX_SLAB_BUSY) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
553 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
554
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
555 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
556 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
557
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
558 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
559 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
560 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
561
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
562 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
563
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
564 if (page->slab) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
565 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
566 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
567
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
568 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
569
7049
94f30939545a Slab: style.
Ruslan Ermilov <ru@nginx.com>
parents: 6861
diff changeset
570 pool->stats[slot].total -= 8 * sizeof(uintptr_t);
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
571
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
572 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
573 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
574
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
575 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
576
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
577 case NGX_SLAB_BIG:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
578
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
579 shift = slab & NGX_SLAB_SHIFT_MASK;
6861
e4590dfd97ff Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents: 6830
diff changeset
580 size = (size_t) 1 << shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
581
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
582 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
583 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
584 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
585
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
586 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
587 + NGX_SLAB_MAP_SHIFT);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
588
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
589 if (slab & m) {
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
590 slot = shift - pool->min_shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
591
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
592 if (page->next == NULL) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
593 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
594
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
595 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
596 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
597
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
598 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
599 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
600 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
601
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
602 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
603
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
604 if (page->slab & NGX_SLAB_MAP_MASK) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
605 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
606 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
607
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
608 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
609
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
610 pool->stats[slot].total -= ngx_pagesize >> shift;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
611
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
612 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
613 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
614
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
615 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
616
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
617 case NGX_SLAB_PAGE:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
618
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
619 if ((uintptr_t) p & (ngx_pagesize - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
620 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
621 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
622
6809
87d7e640b45d Slab: improved double free detection.
Ruslan Ermilov <ru@nginx.com>
parents: 6808
diff changeset
623 if (!(slab & NGX_SLAB_PAGE_START)) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
624 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
625 "ngx_slab_free(): page is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
626 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
627 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
628
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
629 if (slab == NGX_SLAB_PAGE_BUSY) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
630 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
631 "ngx_slab_free(): pointer to wrong page");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
632 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
633 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
634
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
635 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
636 size = slab & ~NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
637
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
638 ngx_slab_free_pages(pool, &pool->pages[n], size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
639
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
640 ngx_slab_junk(p, size << ngx_pagesize_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
641
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
642 return;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
643 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
644
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
645 /* not reached */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
646
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
647 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
648
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
649 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
650
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
651 pool->stats[slot].used--;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
652
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
653 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
654
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
655 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
656
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
657 wrong_chunk:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
658
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
659 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
660 "ngx_slab_free(): pointer to wrong chunk");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
661
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
662 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
663
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
664 chunk_already_free:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
665
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
666 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
667 "ngx_slab_free(): chunk is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
668
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
669 fail:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
670
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
671 return;
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
672 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
673
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
674
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
675 static ngx_slab_page_t *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
676 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
677 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
678 ngx_slab_page_t *page, *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
679
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
680 for (page = pool->free.next; page != &pool->free; page = page->next) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
681
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
682 if (page->slab >= pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
683
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
684 if (page->slab > pages) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
685 page[page->slab - 1].prev = (uintptr_t) &page[pages];
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
686
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
687 page[pages].slab = page->slab - pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
688 page[pages].next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
689 page[pages].prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
690
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
691 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
692 p->next = &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
693 page->next->prev = (uintptr_t) &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
694
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
695 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
696 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
697 p->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
698 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
699 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
700
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
701 page->slab = pages | NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
702 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
703 page->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
704
6829
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
705 pool->pfree -= pages;
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
706
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
707 if (--pages == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
708 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
709 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
710
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
711 for (p = page + 1; pages; pages--) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
712 p->slab = NGX_SLAB_PAGE_BUSY;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
713 p->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
714 p->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
715 p++;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
716 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
717
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
718 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
719 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
720 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
721
5634
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
722 if (pool->log_nomem) {
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
723 ngx_slab_error(pool, NGX_LOG_CRIT,
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
724 "ngx_slab_alloc() failed: no memory");
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
725 }
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
726
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
727 return NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
728 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
729
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
730
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
731 static void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
732 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
733 ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
734 {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
735 ngx_slab_page_t *prev, *join;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
736
6829
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
737 pool->pfree += pages;
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
738
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
739 page->slab = pages--;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
740
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
741 if (pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
742 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
743 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
744
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
745 if (page->next) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
746 prev = ngx_slab_page_prev(page);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
747 prev->next = page->next;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
748 page->next->prev = page->prev;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
749 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
750
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
751 join = page + page->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
752
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
753 if (join < pool->last) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
754
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
755 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
756
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
757 if (join->next != NULL) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
758 pages += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
759 page->slab += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
760
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
761 prev = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
762 prev->next = join->next;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
763 join->next->prev = join->prev;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
764
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
765 join->slab = NGX_SLAB_PAGE_FREE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
766 join->next = NULL;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
767 join->prev = NGX_SLAB_PAGE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
768 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
769 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
770 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
771
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
772 if (page > pool->pages) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
773 join = page - 1;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
774
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
775 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
776
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
777 if (join->slab == NGX_SLAB_PAGE_FREE) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
778 join = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
779 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
780
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
781 if (join->next != NULL) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
782 pages += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
783 join->slab += page->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
784
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
785 prev = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
786 prev->next = join->next;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
787 join->next->prev = join->prev;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
788
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
789 page->slab = NGX_SLAB_PAGE_FREE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
790 page->next = NULL;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
791 page->prev = NGX_SLAB_PAGE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
792
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
793 page = join;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
794 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
795 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
796 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
797
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
798 if (pages) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
799 page[pages].prev = (uintptr_t) page;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
800 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
801
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
802 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
803 page->next = pool->free.next;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
804
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
805 page->next->prev = (uintptr_t) page;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
806
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
807 pool->free.next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
808 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
809
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
810
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
811 static void
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
812 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text)
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
813 {
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
814 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
815 }