annotate src/core/ngx_slab.c @ 6957:83bae3d354ab

HTTP/2: fixed connection finalization. All streams in connection must be finalized before the connection itself can be finalized and all related memory is freed. That's not always possible on the current event loop iteration. Thus when the last stream is finalized, it sets the special read event handler ngx_http_v2_handle_connection_handler() and posts the event. Previously, this handler didn't check the connection state and could call the regular event handler on a connection that was already in finalization stage. In the worst case that could lead to a segmentation fault, since some data structures aren't supposed to be used during connection finalization. Particularly, the waiting queue can contain already freed streams, so the WINDOW_UPDATE frame received by that moment could trigger accessing to these freed streams. Now, the connection error flag is explicitly checked in ngx_http_v2_handle_connection_handler().
author Valentin Bartenev <vbart@nginx.com>
date Wed, 29 Mar 2017 20:21:01 +0300
parents e4590dfd97ff
children 94f30939545a
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
1
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
2 /*
444
42d11f017717 nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents: 441
diff changeset
3 * Copyright (C) Igor Sysoev
4412
d620f497c50f Copyright updated.
Maxim Konovalov <maxim@nginx.com>
parents: 4224
diff changeset
4 * Copyright (C) Nginx, Inc.
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
5 */
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
6
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
7 #include <ngx_config.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
8 #include <ngx_core.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
9
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
10
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
11 #define NGX_SLAB_PAGE_MASK 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
12 #define NGX_SLAB_PAGE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
13 #define NGX_SLAB_BIG 1
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
14 #define NGX_SLAB_EXACT 2
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
15 #define NGX_SLAB_SMALL 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
17 #if (NGX_PTR_SIZE == 4)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
18
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
19 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
20 #define NGX_SLAB_PAGE_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
21 #define NGX_SLAB_PAGE_START 0x80000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
22
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
23 #define NGX_SLAB_SHIFT_MASK 0x0000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
24 #define NGX_SLAB_MAP_MASK 0xffff0000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
25 #define NGX_SLAB_MAP_SHIFT 16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
26
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
27 #define NGX_SLAB_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
28
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
29 #else /* (NGX_PTR_SIZE == 8) */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
30
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
31 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
33 #define NGX_SLAB_PAGE_START 0x8000000000000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
34
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
37 #define NGX_SLAB_MAP_SHIFT 32
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
38
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
39 #define NGX_SLAB_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
40
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
41 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
42
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
43
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
44 #define ngx_slab_slots(pool) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
45 (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t))
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
46
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
47 #define ngx_slab_page_type(page) ((page)->prev & NGX_SLAB_PAGE_MASK)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
48
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
49 #define ngx_slab_page_prev(page) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
50 (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
51
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
52 #define ngx_slab_page_addr(pool, page) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
53 ((((page) - (pool)->pages) << ngx_pagesize_shift) \
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
54 + (uintptr_t) (pool)->start)
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
55
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
56
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
57 #if (NGX_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
58
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
59 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
60
4764
0ed8088f43b4 Reduced the number of preprocessor directives.
Ruslan Ermilov <ru@nginx.com>
parents: 4412
diff changeset
61 #elif (NGX_HAVE_DEBUG_MALLOC)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
62
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
63 #define ngx_slab_junk(p, size) \
4224
4c4997de65b8 Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents: 4223
diff changeset
64 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
65
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
66 #else
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
67
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
68 #define ngx_slab_junk(p, size)
1771
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
69
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
70 #endif
b794d78e52c5 take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents: 1645
diff changeset
71
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
72 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
73 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
74 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
75 ngx_uint_t pages);
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
76 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
77 char *text);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
78
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
79
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
80 static ngx_uint_t ngx_slab_max_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
81 static ngx_uint_t ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
82 static ngx_uint_t ngx_slab_exact_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
83
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
84
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
85 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
86 ngx_slab_init(ngx_slab_pool_t *pool)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
87 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
88 u_char *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
89 size_t size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
90 ngx_int_t m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
91 ngx_uint_t i, n, pages;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
92 ngx_slab_page_t *slots, *page;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
93
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
94 /* STUB */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
95 if (ngx_slab_max_size == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
96 ngx_slab_max_size = ngx_pagesize / 2;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
97 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
98 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
99 /* void */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
100 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
101 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
102 /**/
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
103
6861
e4590dfd97ff Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents: 6830
diff changeset
104 pool->min_size = (size_t) 1 << pool->min_shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
105
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
106 slots = ngx_slab_slots(pool);
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
107
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
108 p = (u_char *) slots;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
109 size = pool->end - p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
110
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
111 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
112
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
113 n = ngx_pagesize_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
114
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
115 for (i = 0; i < n; i++) {
6824
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
116 /* only "next" is used in list head */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
117 slots[i].slab = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
118 slots[i].next = &slots[i];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
119 slots[i].prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
120 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
121
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
122 p += n * sizeof(ngx_slab_page_t);
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
123
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
124 pool->stats = (ngx_slab_stat_t *) p;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
125 ngx_memzero(pool->stats, n * sizeof(ngx_slab_stat_t));
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
126
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
127 p += n * sizeof(ngx_slab_stat_t);
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
128
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
129 size -= n * (sizeof(ngx_slab_page_t) + sizeof(ngx_slab_stat_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
130
1645
31622d9f2c0d the simple expression has the same precision without overflow
Igor Sysoev <igor@sysoev.ru>
parents: 1018
diff changeset
131 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
132
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
133 pool->pages = (ngx_slab_page_t *) p;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
134 ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
135
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
136 page = pool->pages;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
137
6824
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
138 /* only "next" is used in list head */
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
139 pool->free.slab = 0;
ea12328518dc Slab: added comment about list heads.
Ruslan Ermilov <ru@nginx.com>
parents: 6823
diff changeset
140 pool->free.next = page;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
141 pool->free.prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
142
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
143 page->slab = pages;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
144 page->next = &pool->free;
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
145 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
146
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
147 pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t),
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
148 ngx_pagesize);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
149
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
150 m = pages - (pool->end - pool->start) / ngx_pagesize;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
151 if (m > 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
152 pages -= m;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
153 page->slab = pages;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
154 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
155
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
156 pool->last = pool->pages + pages;
6829
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
157 pool->pfree = pages;
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
158
5634
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
159 pool->log_nomem = 1;
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
160 pool->log_ctx = &pool->zero;
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
161 pool->zero = '\0';
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
162 }
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
163
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
164
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
165 void *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
166 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
167 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
168 void *p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
169
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
170 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
171
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
172 p = ngx_slab_alloc_locked(pool, size);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
173
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
174 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
175
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
176 return p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
177 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
178
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
179
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
180 void *
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
181 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
182 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
183 size_t s;
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
184 uintptr_t p, n, m, mask, *bitmap;
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
185 ngx_uint_t i, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
186 ngx_slab_page_t *page, *prev, *slots;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
187
5766
abdb027be9d5 Core: bugfix for the ngx_slab_max_size case.
Jianjun Zheng <codeeply@gmail.com>
parents: 5726
diff changeset
188 if (size > ngx_slab_max_size) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
189
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
190 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
967
1cb178720355 style fix
Igor Sysoev <igor@sysoev.ru>
parents: 966
diff changeset
191 "slab alloc: %uz", size);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
192
4829
40de49cf6b37 Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents: 4764
diff changeset
193 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
40de49cf6b37 Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents: 4764
diff changeset
194 + ((size % ngx_pagesize) ? 1 : 0));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
195 if (page) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
196 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
197
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
198 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
199 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
200 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
201
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
202 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
203 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
204
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
205 if (size > pool->min_size) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
206 shift = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
207 for (s = size - 1; s >>= 1; shift++) { /* void */ }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
208 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
209
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
210 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
211 shift = pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
212 slot = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
213 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
214
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
215 pool->stats[slot].reqs++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
216
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
217 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
218 "slab alloc: %uz slot: %ui", size, slot);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
219
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
220 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
221 page = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
222
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
223 if (page->next != page) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
224
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
225 if (shift < ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
226
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
227 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
228
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
229 map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
230
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
231 for (n = 0; n < map; n++) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
232
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
233 if (bitmap[n] != NGX_SLAB_BUSY) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
234
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
235 for (m = 1, i = 0; m; m <<= 1, i++) {
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
236 if (bitmap[n] & m) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
237 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
238 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
239
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
240 bitmap[n] |= m;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
241
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
242 i = (n * sizeof(uintptr_t) * 8 + i) << shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
243
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
244 p = (uintptr_t) bitmap + i;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
245
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
246 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
247
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
248 if (bitmap[n] == NGX_SLAB_BUSY) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
249 for (n = n + 1; n < map; n++) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
250 if (bitmap[n] != NGX_SLAB_BUSY) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
251 goto done;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
252 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
253 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
254
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
255 prev = ngx_slab_page_prev(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
256 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
257 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
258
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
259 page->next = NULL;
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
260 page->prev = NGX_SLAB_SMALL;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
261 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
262
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
263 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
264 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
265 }
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
266 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
267
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
268 } else if (shift == ngx_slab_exact_shift) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
269
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
270 for (m = 1, i = 0; m; m <<= 1, i++) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
271 if (page->slab & m) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
272 continue;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
273 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
274
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
275 page->slab |= m;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
276
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
277 if (page->slab == NGX_SLAB_BUSY) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
278 prev = ngx_slab_page_prev(page);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
279 prev->next = page->next;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
280 page->next->prev = page->prev;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
281
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
282 page->next = NULL;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
283 page->prev = NGX_SLAB_EXACT;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
284 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
285
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
286 p = ngx_slab_page_addr(pool, page) + (i << shift);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
287
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
288 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
289
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
290 goto done;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
291 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
292
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
293 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
294
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
295 mask = ((uintptr_t) 1 << (ngx_pagesize >> shift)) - 1;
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
296 mask <<= NGX_SLAB_MAP_SHIFT;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
297
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
298 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
299 m & mask;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
300 m <<= 1, i++)
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
301 {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
302 if (page->slab & m) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
303 continue;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
304 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
305
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
306 page->slab |= m;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
307
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
308 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
309 prev = ngx_slab_page_prev(page);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
310 prev->next = page->next;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
311 page->next->prev = page->prev;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
312
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
313 page->next = NULL;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
314 page->prev = NGX_SLAB_BIG;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
315 }
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
316
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
317 p = ngx_slab_page_addr(pool, page) + (i << shift);
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
318
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
319 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
320
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
321 goto done;
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
322 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
323 }
6826
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
324
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
325 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_alloc(): page is busy");
d0404c9a7675 Slab: simplified allocation from slots.
Ruslan Ermilov <ru@nginx.com>
parents: 6825
diff changeset
326 ngx_debug_point();
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
327 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
328
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
329 page = ngx_slab_alloc_pages(pool, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
330
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
331 if (page) {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
332 if (shift < ngx_slab_exact_shift) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
333 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
334
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
335 n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
336
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
337 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
338 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
339 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
340
6830
6eed5ed31e22 Slab: commented bitmap initialization for small allocations.
Ruslan Ermilov <ru@nginx.com>
parents: 6829
diff changeset
341 /* "n" elements for bitmap, plus one requested */
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
342 bitmap[0] = ((uintptr_t) 2 << n) - 1;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
343
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
344 map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
345
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
346 for (i = 1; i < map; i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
347 bitmap[i] = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
348 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
349
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
350 page->slab = shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
351 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
352 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
353
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
354 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
355
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
356 pool->stats[slot].total += (ngx_pagesize >> shift) - n;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
357
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
358 p = ngx_slab_page_addr(pool, page) + (n << shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
359
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
360 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
361
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
362 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
363
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
364 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
365
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
366 page->slab = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
367 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
368 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
369
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
370 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
371
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
372 pool->stats[slot].total += sizeof(uintptr_t) * 8;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
373
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
374 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
375
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
376 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
377
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
378 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
379
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
380 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
381
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
382 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
383 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
384 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
385
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
386 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
387
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
388 pool->stats[slot].total += ngx_pagesize >> shift;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
389
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
390 p = ngx_slab_page_addr(pool, page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
391
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
392 pool->stats[slot].used++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
393
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
394 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
395 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
396 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
397
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
398 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
399
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
400 pool->stats[slot].fails++;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
401
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
402 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
403
6480
f01ab2dbcfdc Fixed logging.
Sergey Kandaurov <pluknet@nginx.com>
parents: 6474
diff changeset
404 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
f01ab2dbcfdc Fixed logging.
Sergey Kandaurov <pluknet@nginx.com>
parents: 6474
diff changeset
405 "slab alloc: %p", (void *) p);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
406
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
407 return (void *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
408 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
409
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
410
5726
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
411 void *
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
412 ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size)
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
413 {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
414 void *p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
415
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
416 ngx_shmtx_lock(&pool->mutex);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
417
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
418 p = ngx_slab_calloc_locked(pool, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
419
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
420 ngx_shmtx_unlock(&pool->mutex);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
421
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
422 return p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
423 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
424
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
425
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
426 void *
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
427 ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size)
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
428 {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
429 void *p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
430
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
431 p = ngx_slab_alloc_locked(pool, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
432 if (p) {
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
433 ngx_memzero(p, size);
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
434 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
435
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
436 return p;
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
437 }
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
438
25ade23cf281 Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents: 5718
diff changeset
439
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
440 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
441 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
442 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
443 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
444
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
445 ngx_slab_free_locked(pool, p);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
446
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
447 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
448 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
449
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
450
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
451 void
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
452 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
453 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
454 size_t size;
1018
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
455 uintptr_t slab, m, *bitmap;
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
456 ngx_uint_t i, n, type, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
457 ngx_slab_page_t *slots, *page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
458
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
459 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
460
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
461 if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
462 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
463 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
464 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
465
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
466 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
467 page = &pool->pages[n];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
468 slab = page->slab;
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
469 type = ngx_slab_page_type(page);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
470
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
471 switch (type) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
472
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
473 case NGX_SLAB_SMALL:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
474
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
475 shift = slab & NGX_SLAB_SHIFT_MASK;
6861
e4590dfd97ff Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents: 6830
diff changeset
476 size = (size_t) 1 << shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
477
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
478 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
479 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
480 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
481
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
482 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
483 m = (uintptr_t) 1 << (n % (sizeof(uintptr_t) * 8));
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
484 n /= sizeof(uintptr_t) * 8;
5362
79b9101cecf4 Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents: 4829
diff changeset
485 bitmap = (uintptr_t *)
79b9101cecf4 Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents: 4829
diff changeset
486 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
487
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
488 if (bitmap[n] & m) {
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
489 slot = shift - pool->min_shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
490
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
491 if (page->next == NULL) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
492 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
493
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
494 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
495 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
496
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
497 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
498 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
499 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
500
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
501 bitmap[n] &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
502
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
503 n = (ngx_pagesize >> shift) / ((1 << shift) * 8);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
504
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
505 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
506 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
507 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
508
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
509 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
510 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
511 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
512
6827
0e61510c56c4 Slab: simplified some math.
Ruslan Ermilov <ru@nginx.com>
parents: 6826
diff changeset
513 map = (ngx_pagesize >> shift) / (sizeof(uintptr_t) * 8);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
514
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
515 for (i = 1; i < map; i++) {
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
516 if (bitmap[i]) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
517 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
518 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
519 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
520
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
521 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
522
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
523 pool->stats[slot].total -= (ngx_pagesize >> shift) - n;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
524
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
525 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
526 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
527
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
528 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
529
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
530 case NGX_SLAB_EXACT:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
531
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
532 m = (uintptr_t) 1 <<
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
533 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
534 size = ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
535
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
536 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
537 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
538 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
539
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
540 if (slab & m) {
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
541 slot = ngx_slab_exact_shift - pool->min_shift;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
542
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
543 if (slab == NGX_SLAB_BUSY) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
544 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
545
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
546 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
547 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
548
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
549 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
550 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
551 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
552
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
553 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
554
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
555 if (page->slab) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
556 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
557 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
558
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
559 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
560
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
561 pool->stats[slot].total -= sizeof(uintptr_t) * 8;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
562
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
563 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
564 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
565
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
566 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
567
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
568 case NGX_SLAB_BIG:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
569
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
570 shift = slab & NGX_SLAB_SHIFT_MASK;
6861
e4590dfd97ff Win32: support 64-bit compilation with MSVC.
Maxim Dounin <mdounin@mdounin.ru>
parents: 6830
diff changeset
571 size = (size_t) 1 << shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
572
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
573 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
574 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
575 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
576
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
577 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
578 + NGX_SLAB_MAP_SHIFT);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
579
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
580 if (slab & m) {
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
581 slot = shift - pool->min_shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
582
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
583 if (page->next == NULL) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
584 slots = ngx_slab_slots(pool);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
585
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
586 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
587 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
588
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
589 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
590 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
591 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
592
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
593 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
594
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
595 if (page->slab & NGX_SLAB_MAP_MASK) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
596 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
597 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
598
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
599 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
600
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
601 pool->stats[slot].total -= ngx_pagesize >> shift;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
602
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
603 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
604 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
605
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
606 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
607
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
608 case NGX_SLAB_PAGE:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
609
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
610 if ((uintptr_t) p & (ngx_pagesize - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
611 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
612 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
613
6809
87d7e640b45d Slab: improved double free detection.
Ruslan Ermilov <ru@nginx.com>
parents: 6808
diff changeset
614 if (!(slab & NGX_SLAB_PAGE_START)) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
615 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
616 "ngx_slab_free(): page is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
617 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
618 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
619
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
620 if (slab == NGX_SLAB_PAGE_BUSY) {
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
621 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
622 "ngx_slab_free(): pointer to wrong page");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
623 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
624 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
625
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
626 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
627 size = slab & ~NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
628
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
629 ngx_slab_free_pages(pool, &pool->pages[n], size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
630
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
631 ngx_slab_junk(p, size << ngx_pagesize_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
632
2711
6b87e875e87a fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents: 2611
diff changeset
633 return;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
634 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
635
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
636 /* not reached */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
637
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
638 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
639
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
640 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
641
6828
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
642 pool->stats[slot].used--;
99770a5ea14f Slab: slots statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6827
diff changeset
643
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
644 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
645
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
646 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
647
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
648 wrong_chunk:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
649
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
650 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
651 "ngx_slab_free(): pointer to wrong chunk");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
652
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
653 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
654
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
655 chunk_already_free:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
656
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
657 ngx_slab_error(pool, NGX_LOG_ALERT,
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
658 "ngx_slab_free(): chunk is already free");
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
659
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
660 fail:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
661
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
662 return;
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
663 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
664
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
665
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
666 static ngx_slab_page_t *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
667 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
668 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
669 ngx_slab_page_t *page, *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
670
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
671 for (page = pool->free.next; page != &pool->free; page = page->next) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
672
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
673 if (page->slab >= pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
674
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
675 if (page->slab > pages) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
676 page[page->slab - 1].prev = (uintptr_t) &page[pages];
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
677
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
678 page[pages].slab = page->slab - pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
679 page[pages].next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
680 page[pages].prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
681
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
682 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
683 p->next = &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
684 page->next->prev = (uintptr_t) &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
685
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
686 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
687 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
688 p->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
689 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
690 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
691
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
692 page->slab = pages | NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
693 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
694 page->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
695
6829
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
696 pool->pfree -= pages;
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
697
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
698 if (--pages == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
699 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
700 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
701
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
702 for (p = page + 1; pages; pages--) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
703 p->slab = NGX_SLAB_PAGE_BUSY;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
704 p->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
705 p->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
706 p++;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
707 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
708
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
709 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
710 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
711 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
712
5634
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
713 if (pool->log_nomem) {
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
714 ngx_slab_error(pool, NGX_LOG_CRIT,
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
715 "ngx_slab_alloc() failed: no memory");
5024d29354f1 Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5362
diff changeset
716 }
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
717
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
718 return NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
719 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
720
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
721
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
722 static void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
723 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
724 ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
725 {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
726 ngx_slab_page_t *prev, *join;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
727
6829
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
728 pool->pfree += pages;
6e757036e588 Slab: free pages statistics.
Ruslan Ermilov <ru@nginx.com>
parents: 6828
diff changeset
729
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
730 page->slab = pages--;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
731
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
732 if (pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
733 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
734 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
735
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
736 if (page->next) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
737 prev = ngx_slab_page_prev(page);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
738 prev->next = page->next;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
739 page->next->prev = page->prev;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
740 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
741
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
742 join = page + page->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
743
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
744 if (join < pool->last) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
745
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
746 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
747
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
748 if (join->next != NULL) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
749 pages += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
750 page->slab += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
751
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
752 prev = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
753 prev->next = join->next;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
754 join->next->prev = join->prev;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
755
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
756 join->slab = NGX_SLAB_PAGE_FREE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
757 join->next = NULL;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
758 join->prev = NGX_SLAB_PAGE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
759 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
760 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
761 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
762
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
763 if (page > pool->pages) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
764 join = page - 1;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
765
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
766 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
767
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
768 if (join->slab == NGX_SLAB_PAGE_FREE) {
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
769 join = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
770 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
771
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
772 if (join->next != NULL) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
773 pages += join->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
774 join->slab += page->slab;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
775
6823
88c8c3d65184 Slab: improved code readability.
Ruslan Ermilov <ru@nginx.com>
parents: 6809
diff changeset
776 prev = ngx_slab_page_prev(join);
5718
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
777 prev->next = join->next;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
778 join->next->prev = join->prev;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
779
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
780 page->slab = NGX_SLAB_PAGE_FREE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
781 page->next = NULL;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
782 page->prev = NGX_SLAB_PAGE;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
783
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
784 page = join;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
785 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
786 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
787 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
788
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
789 if (pages) {
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
790 page[pages].prev = (uintptr_t) page;
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
791 }
c46657e391a3 Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents: 5634
diff changeset
792
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
793 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
794 page->next = pool->free.next;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
795
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
796 page->next->prev = (uintptr_t) page;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
797
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
798 pool->free.next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
799 }
2611
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
800
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
801
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
802 static void
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
803 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text)
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
804 {
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
805 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx);
2bce3f6416c6 improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents: 2610
diff changeset
806 }