Mercurial > hg > nginx-quic
annotate src/core/ngx_slab.c @ 5114:a46c1cfe7263
Allow to reuse connections that wait their first request.
This should improve behavior under deficiency of connections.
Since SSL handshake usually takes significant amount of time,
we exclude connections from reusable queue during this period
to avoid premature flush of them.
author | Valentin Bartenev <vbart@nginx.com> |
---|---|
date | Fri, 15 Mar 2013 19:49:54 +0000 |
parents | 40de49cf6b37 |
children | 79b9101cecf4 |
rev | line source |
---|---|
441
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
1 |
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
2 /* |
444
42d11f017717
nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents:
441
diff
changeset
|
3 * Copyright (C) Igor Sysoev |
4412 | 4 * Copyright (C) Nginx, Inc. |
441
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
5 */ |
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
6 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
7 #include <ngx_config.h> |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
8 #include <ngx_core.h> |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
9 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
10 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
11 #define NGX_SLAB_PAGE_MASK 3 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
12 #define NGX_SLAB_PAGE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
13 #define NGX_SLAB_BIG 1 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
14 #define NGX_SLAB_EXACT 2 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
15 #define NGX_SLAB_SMALL 3 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
16 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
17 #if (NGX_PTR_SIZE == 4) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
18 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
19 #define NGX_SLAB_PAGE_FREE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
20 #define NGX_SLAB_PAGE_BUSY 0xffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
21 #define NGX_SLAB_PAGE_START 0x80000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
22 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
23 #define NGX_SLAB_SHIFT_MASK 0x0000000f |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
24 #define NGX_SLAB_MAP_MASK 0xffff0000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
25 #define NGX_SLAB_MAP_SHIFT 16 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
26 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
27 #define NGX_SLAB_BUSY 0xffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
28 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
29 #else /* (NGX_PTR_SIZE == 8) */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
30 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
31 #define NGX_SLAB_PAGE_FREE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
33 #define NGX_SLAB_PAGE_START 0x8000000000000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
34 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
37 #define NGX_SLAB_MAP_SHIFT 32 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
38 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
39 #define NGX_SLAB_BUSY 0xffffffffffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
40 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
41 #endif |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
42 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
43 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
44 #if (NGX_DEBUG_MALLOC) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
45 |
4224
4c4997de65b8
Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents:
4223
diff
changeset
|
46 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
47 |
4764
0ed8088f43b4
Reduced the number of preprocessor directives.
Ruslan Ermilov <ru@nginx.com>
parents:
4412
diff
changeset
|
48 #elif (NGX_HAVE_DEBUG_MALLOC) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
49 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
50 #define ngx_slab_junk(p, size) \ |
4224
4c4997de65b8
Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents:
4223
diff
changeset
|
51 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
52 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
53 #else |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
54 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
55 #define ngx_slab_junk(p, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
56 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
57 #endif |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
58 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
59 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
60 ngx_uint_t pages); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
61 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
62 ngx_uint_t pages); |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
63 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
64 char *text); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
65 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
66 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
67 static ngx_uint_t ngx_slab_max_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
68 static ngx_uint_t ngx_slab_exact_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
69 static ngx_uint_t ngx_slab_exact_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
70 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
71 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
72 void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
73 ngx_slab_init(ngx_slab_pool_t *pool) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
74 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
75 u_char *p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
76 size_t size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
77 ngx_int_t m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
78 ngx_uint_t i, n, pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
79 ngx_slab_page_t *slots; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
80 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
81 /* STUB */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
82 if (ngx_slab_max_size == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
83 ngx_slab_max_size = ngx_pagesize / 2; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
84 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
85 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
86 /* void */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
87 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
88 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
89 /**/ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
90 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
91 pool->min_size = 1 << pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
92 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
93 p = (u_char *) pool + sizeof(ngx_slab_pool_t); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
94 size = pool->end - p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
95 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
96 ngx_slab_junk(p, size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
97 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
98 slots = (ngx_slab_page_t *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
99 n = ngx_pagesize_shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
100 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
101 for (i = 0; i < n; i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
102 slots[i].slab = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
103 slots[i].next = &slots[i]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
104 slots[i].prev = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
105 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
106 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
107 p += n * sizeof(ngx_slab_page_t); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
108 |
1645
31622d9f2c0d
the simple expression has the same precision without overflow
Igor Sysoev <igor@sysoev.ru>
parents:
1018
diff
changeset
|
109 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t))); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
110 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
111 ngx_memzero(p, pages * sizeof(ngx_slab_page_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
112 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
113 pool->pages = (ngx_slab_page_t *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
114 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
115 pool->free.prev = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
116 pool->free.next = (ngx_slab_page_t *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
117 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
118 pool->pages->slab = pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
119 pool->pages->next = &pool->free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
120 pool->pages->prev = (uintptr_t) &pool->free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
121 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
122 pool->start = (u_char *) |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
123 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t), |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
124 ngx_pagesize); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
125 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
126 m = pages - (pool->end - pool->start) / ngx_pagesize; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
127 if (m > 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
128 pages -= m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
129 pool->pages->slab = pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
130 } |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
131 |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
132 pool->log_ctx = &pool->zero; |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
133 pool->zero = '\0'; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
134 } |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
135 |
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
136 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
137 void * |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
138 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
139 { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
140 void *p; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
141 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
142 ngx_shmtx_lock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
143 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
144 p = ngx_slab_alloc_locked(pool, size); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
145 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
146 ngx_shmtx_unlock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
147 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
148 return p; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
149 } |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
150 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
151 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
152 void * |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
153 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
154 { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
155 size_t s; |
881
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
156 uintptr_t p, n, m, mask, *bitmap; |
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
157 ngx_uint_t i, slot, shift, map; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
158 ngx_slab_page_t *page, *prev, *slots; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
159 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
160 if (size >= ngx_slab_max_size) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
161 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
162 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
967 | 163 "slab alloc: %uz", size); |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
164 |
4829
40de49cf6b37
Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents:
4764
diff
changeset
|
165 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift) |
40de49cf6b37
Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents:
4764
diff
changeset
|
166 + ((size % ngx_pagesize) ? 1 : 0)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
167 if (page) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
168 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
169 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
170 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
171 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
172 p = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
173 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
174 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
175 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
176 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
177 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
178 if (size > pool->min_size) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
179 shift = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
180 for (s = size - 1; s >>= 1; shift++) { /* void */ } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
181 slot = shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
182 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
183 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
184 size = pool->min_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
185 shift = pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
186 slot = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
187 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
188 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
189 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
190 "slab alloc: %uz slot: %ui", size, slot); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
191 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
192 slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
193 page = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
194 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
195 if (page->next != page) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
196 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
197 if (shift < ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
198 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
199 do { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
200 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
201 bitmap = (uintptr_t *) (pool->start + p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
202 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
203 map = (1 << (ngx_pagesize_shift - shift)) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
204 / (sizeof(uintptr_t) * 8); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
205 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
206 for (n = 0; n < map; n++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
207 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
208 if (bitmap[n] != NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
209 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
210 for (m = 1, i = 0; m; m <<= 1, i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
211 if ((bitmap[n] & m)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
212 continue; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
213 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
214 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
215 bitmap[n] |= m; |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
216 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
217 i = ((n * sizeof(uintptr_t) * 8) << shift) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
218 + (i << shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
219 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
220 if (bitmap[n] == NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
221 for (n = n + 1; n < map; n++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
222 if (bitmap[n] != NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
223 p = (uintptr_t) bitmap + i; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
224 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
225 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
226 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
227 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
228 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
229 prev = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
230 (page->prev & ~NGX_SLAB_PAGE_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
231 prev->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
232 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
233 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
234 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
235 page->prev = NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
236 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
237 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
238 p = (uintptr_t) bitmap + i; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
239 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
240 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
241 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
242 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
243 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
244 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
245 page = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
246 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
247 } while (page); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
248 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
249 } else if (shift == ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
250 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
251 do { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
252 if (page->slab != NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
253 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
254 for (m = 1, i = 0; m; m <<= 1, i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
255 if ((page->slab & m)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
256 continue; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
257 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
258 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
259 page->slab |= m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
260 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
261 if (page->slab == NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
262 prev = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
263 (page->prev & ~NGX_SLAB_PAGE_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
264 prev->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
265 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
266 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
267 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
268 page->prev = NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
269 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
270 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
271 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
272 p += i << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
273 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
274 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
275 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
276 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
277 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
278 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
279 page = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
280 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
281 } while (page); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
282 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
283 } else { /* shift > ngx_slab_exact_shift */ |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
284 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
285 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
286 n = 1 << n; |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
287 n = ((uintptr_t) 1 << n) - 1; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
288 mask = n << NGX_SLAB_MAP_SHIFT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
289 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
290 do { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
291 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
292 |
881
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
293 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
294 m & mask; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
295 m <<= 1, i++) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
296 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
297 if ((page->slab & m)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
298 continue; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
299 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
300 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
301 page->slab |= m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
302 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
303 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
304 prev = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
305 (page->prev & ~NGX_SLAB_PAGE_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
306 prev->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
307 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
308 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
309 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
310 page->prev = NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
311 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
312 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
313 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
314 p += i << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
315 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
316 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
317 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
318 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
319 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
320 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
321 page = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
322 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
323 } while (page); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
324 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
325 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
326 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
327 page = ngx_slab_alloc_pages(pool, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
328 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
329 if (page) { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
330 if (shift < ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
331 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
332 bitmap = (uintptr_t *) (pool->start + p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
333 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
334 s = 1 << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
335 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
336 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
337 if (n == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
338 n = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
339 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
340 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
341 bitmap[0] = (2 << n) - 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
342 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
343 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
344 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
345 for (i = 1; i < map; i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
346 bitmap[i] = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
347 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
348 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
349 page->slab = shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
350 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
351 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
352 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
353 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
354 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
355 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
356 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
357 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
358 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
359 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
360 } else if (shift == ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
361 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
362 page->slab = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
363 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
364 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
365 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
366 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
367 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
368 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
369 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
370 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
371 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
372 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
373 } else { /* shift > ngx_slab_exact_shift */ |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
374 |
881
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
375 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
376 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
377 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
378 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
379 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
380 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
381 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
382 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
383 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
384 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
385 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
386 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
387 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
388 p = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
389 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
390 done: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
391 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
392 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab alloc: %p", p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
393 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
394 return (void *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
395 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
396 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
397 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
398 void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
399 ngx_slab_free(ngx_slab_pool_t *pool, void *p) |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
400 { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
401 ngx_shmtx_lock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
402 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
403 ngx_slab_free_locked(pool, p); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
404 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
405 ngx_shmtx_unlock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
406 } |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
407 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
408 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
409 void |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
410 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
411 { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
412 size_t size; |
1018
06306a20b8bb
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
1016
diff
changeset
|
413 uintptr_t slab, m, *bitmap; |
06306a20b8bb
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
1016
diff
changeset
|
414 ngx_uint_t n, type, slot, shift, map; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
415 ngx_slab_page_t *slots, *page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
416 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
417 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
418 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
419 if ((u_char *) p < pool->start || (u_char *) p > pool->end) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
420 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
421 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
422 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
423 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
424 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
425 page = &pool->pages[n]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
426 slab = page->slab; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
427 type = page->prev & NGX_SLAB_PAGE_MASK; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
428 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
429 switch (type) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
430 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
431 case NGX_SLAB_SMALL: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
432 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
433 shift = slab & NGX_SLAB_SHIFT_MASK; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
434 size = 1 << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
435 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
436 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
437 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
438 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
439 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
440 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift; |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
441 m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
442 n /= (sizeof(uintptr_t) * 8); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
443 bitmap = (uintptr_t *) ((uintptr_t) p & ~(ngx_pagesize - 1)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
444 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
445 if (bitmap[n] & m) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
446 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
447 if (page->next == NULL) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
448 slots = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
449 ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
450 slot = shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
451 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
452 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
453 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
454 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
455 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
456 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
457 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
458 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
459 bitmap[n] &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
460 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
461 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
462 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
463 if (n == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
464 n = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
465 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
466 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
467 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
468 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
469 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
470 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
471 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
472 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
473 for (n = 1; n < map; n++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
474 if (bitmap[n]) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
475 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
476 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
477 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
478 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
479 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
480 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
481 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
482 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
483 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
484 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
485 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
486 case NGX_SLAB_EXACT: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
487 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
488 m = (uintptr_t) 1 << |
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
489 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
490 size = ngx_slab_exact_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
491 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
492 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
493 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
494 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
495 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
496 if (slab & m) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
497 if (slab == NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
498 slots = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
499 ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
500 slot = ngx_slab_exact_shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
501 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
502 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
503 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
504 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
505 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
506 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
507 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
508 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
509 page->slab &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
510 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
511 if (page->slab) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
512 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
513 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
514 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
515 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
516 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
517 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
518 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
519 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
520 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
521 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
522 case NGX_SLAB_BIG: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
523 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
524 shift = slab & NGX_SLAB_SHIFT_MASK; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
525 size = 1 << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
526 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
527 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
528 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
529 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
530 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
531 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift) |
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
532 + NGX_SLAB_MAP_SHIFT); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
533 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
534 if (slab & m) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
535 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
536 if (page->next == NULL) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
537 slots = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
538 ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
539 slot = shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
540 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
541 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
542 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
543 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
544 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
545 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
546 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
547 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
548 page->slab &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
549 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
550 if (page->slab & NGX_SLAB_MAP_MASK) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
551 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
552 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
553 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
554 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
555 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
556 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
557 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
558 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
559 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
560 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
561 case NGX_SLAB_PAGE: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
562 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
563 if ((uintptr_t) p & (ngx_pagesize - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
564 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
565 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
566 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
567 if (slab == NGX_SLAB_PAGE_FREE) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
568 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
569 "ngx_slab_free(): page is already free"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
570 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
571 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
572 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
573 if (slab == NGX_SLAB_PAGE_BUSY) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
574 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
575 "ngx_slab_free(): pointer to wrong page"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
576 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
577 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
578 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
579 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
580 size = slab & ~NGX_SLAB_PAGE_START; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
581 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
582 ngx_slab_free_pages(pool, &pool->pages[n], size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
583 |
2711
6b87e875e87a
fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents:
2611
diff
changeset
|
584 ngx_slab_junk(p, size << ngx_pagesize_shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
585 |
2711
6b87e875e87a
fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents:
2611
diff
changeset
|
586 return; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
587 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
588 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
589 /* not reached */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
590 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
591 return; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
592 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
593 done: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
594 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
595 ngx_slab_junk(p, size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
596 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
597 return; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
598 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
599 wrong_chunk: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
600 |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
601 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
602 "ngx_slab_free(): pointer to wrong chunk"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
603 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
604 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
605 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
606 chunk_already_free: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
607 |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
608 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
609 "ngx_slab_free(): chunk is already free"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
610 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
611 fail: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
612 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
613 return; |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
614 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
615 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
616 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
617 static ngx_slab_page_t * |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
618 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
619 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
620 ngx_slab_page_t *page, *p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
621 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
622 for (page = pool->free.next; page != &pool->free; page = page->next) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
623 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
624 if (page->slab >= pages) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
625 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
626 if (page->slab > pages) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
627 page[pages].slab = page->slab - pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
628 page[pages].next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
629 page[pages].prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
630 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
631 p = (ngx_slab_page_t *) page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
632 p->next = &page[pages]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
633 page->next->prev = (uintptr_t) &page[pages]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
634 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
635 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
636 p = (ngx_slab_page_t *) page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
637 p->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
638 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
639 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
640 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
641 page->slab = pages | NGX_SLAB_PAGE_START; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
642 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
643 page->prev = NGX_SLAB_PAGE; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
644 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
645 if (--pages == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
646 return page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
647 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
648 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
649 for (p = page + 1; pages; pages--) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
650 p->slab = NGX_SLAB_PAGE_BUSY; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
651 p->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
652 p->prev = NGX_SLAB_PAGE; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
653 p++; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
654 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
655 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
656 return page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
657 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
658 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
659 |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
660 ngx_slab_error(pool, NGX_LOG_CRIT, "ngx_slab_alloc() failed: no memory"); |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
661 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
662 return NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
663 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
664 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
665 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
666 static void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
667 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
668 ngx_uint_t pages) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
669 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
670 ngx_slab_page_t *prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
671 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
672 page->slab = pages--; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
673 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
674 if (pages) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
675 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
676 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
677 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
678 if (page->next) { |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
679 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
680 prev->next = page->next; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
681 page->next->prev = page->prev; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
682 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
683 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
684 page->prev = (uintptr_t) &pool->free; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
685 page->next = pool->free.next; |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
686 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
687 page->next->prev = (uintptr_t) page; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
688 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
689 pool->free.next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
690 } |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
691 |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
692 |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
693 static void |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
694 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text) |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
695 { |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
696 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx); |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
697 } |