Mercurial > hg > nginx-quic
annotate src/core/ngx_slab.c @ 5905:2f7e557eab5b
Cache: proxy_cache_lock_age and friends.
Once this age is reached, the cache lock is discarded and another
request can acquire the lock. Requests which failed to acquire
the lock are not allowed to cache the response.
author | Roman Arutyunyan <arut@nginx.com> |
---|---|
date | Tue, 18 Nov 2014 20:41:12 +0300 |
parents | abdb027be9d5 |
children | 2cd019520210 |
rev | line source |
---|---|
441
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
1 |
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
2 /* |
444
42d11f017717
nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents:
441
diff
changeset
|
3 * Copyright (C) Igor Sysoev |
4412 | 4 * Copyright (C) Nginx, Inc. |
441
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
5 */ |
da8c5707af39
nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents:
357
diff
changeset
|
6 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
7 #include <ngx_config.h> |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
8 #include <ngx_core.h> |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
9 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
10 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
11 #define NGX_SLAB_PAGE_MASK 3 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
12 #define NGX_SLAB_PAGE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
13 #define NGX_SLAB_BIG 1 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
14 #define NGX_SLAB_EXACT 2 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
15 #define NGX_SLAB_SMALL 3 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
16 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
17 #if (NGX_PTR_SIZE == 4) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
18 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
19 #define NGX_SLAB_PAGE_FREE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
20 #define NGX_SLAB_PAGE_BUSY 0xffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
21 #define NGX_SLAB_PAGE_START 0x80000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
22 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
23 #define NGX_SLAB_SHIFT_MASK 0x0000000f |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
24 #define NGX_SLAB_MAP_MASK 0xffff0000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
25 #define NGX_SLAB_MAP_SHIFT 16 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
26 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
27 #define NGX_SLAB_BUSY 0xffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
28 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
29 #else /* (NGX_PTR_SIZE == 8) */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
30 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
31 #define NGX_SLAB_PAGE_FREE 0 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
32 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
33 #define NGX_SLAB_PAGE_START 0x8000000000000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
34 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
35 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
36 #define NGX_SLAB_MAP_MASK 0xffffffff00000000 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
37 #define NGX_SLAB_MAP_SHIFT 32 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
38 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
39 #define NGX_SLAB_BUSY 0xffffffffffffffff |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
40 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
41 #endif |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
42 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
43 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
44 #if (NGX_DEBUG_MALLOC) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
45 |
4224
4c4997de65b8
Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents:
4223
diff
changeset
|
46 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
47 |
4764
0ed8088f43b4
Reduced the number of preprocessor directives.
Ruslan Ermilov <ru@nginx.com>
parents:
4412
diff
changeset
|
48 #elif (NGX_HAVE_DEBUG_MALLOC) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
49 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
50 #define ngx_slab_junk(p, size) \ |
4224
4c4997de65b8
Using of junk value in slab allocator similar to modern FreeBSD values.
Igor Sysoev <igor@sysoev.ru>
parents:
4223
diff
changeset
|
51 if (ngx_debug_malloc) ngx_memset(p, 0xA5, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
52 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
53 #else |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
54 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
55 #define ngx_slab_junk(p, size) |
1771
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
56 |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
57 #endif |
b794d78e52c5
take MALLOC_OPTIONS=J into account in slab allocator
Igor Sysoev <igor@sysoev.ru>
parents:
1645
diff
changeset
|
58 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
59 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
60 ngx_uint_t pages); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
61 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
62 ngx_uint_t pages); |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
63 static void ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
64 char *text); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
65 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
66 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
67 static ngx_uint_t ngx_slab_max_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
68 static ngx_uint_t ngx_slab_exact_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
69 static ngx_uint_t ngx_slab_exact_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
70 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
71 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
72 void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
73 ngx_slab_init(ngx_slab_pool_t *pool) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
74 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
75 u_char *p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
76 size_t size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
77 ngx_int_t m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
78 ngx_uint_t i, n, pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
79 ngx_slab_page_t *slots; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
80 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
81 /* STUB */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
82 if (ngx_slab_max_size == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
83 ngx_slab_max_size = ngx_pagesize / 2; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
84 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
85 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
86 /* void */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
87 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
88 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
89 /**/ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
90 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
91 pool->min_size = 1 << pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
92 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
93 p = (u_char *) pool + sizeof(ngx_slab_pool_t); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
94 size = pool->end - p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
95 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
96 ngx_slab_junk(p, size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
97 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
98 slots = (ngx_slab_page_t *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
99 n = ngx_pagesize_shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
100 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
101 for (i = 0; i < n; i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
102 slots[i].slab = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
103 slots[i].next = &slots[i]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
104 slots[i].prev = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
105 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
106 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
107 p += n * sizeof(ngx_slab_page_t); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
108 |
1645
31622d9f2c0d
the simple expression has the same precision without overflow
Igor Sysoev <igor@sysoev.ru>
parents:
1018
diff
changeset
|
109 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t))); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
110 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
111 ngx_memzero(p, pages * sizeof(ngx_slab_page_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
112 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
113 pool->pages = (ngx_slab_page_t *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
114 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
115 pool->free.prev = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
116 pool->free.next = (ngx_slab_page_t *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
117 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
118 pool->pages->slab = pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
119 pool->pages->next = &pool->free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
120 pool->pages->prev = (uintptr_t) &pool->free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
121 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
122 pool->start = (u_char *) |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
123 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t), |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
124 ngx_pagesize); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
125 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
126 m = pages - (pool->end - pool->start) / ngx_pagesize; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
127 if (m > 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
128 pages -= m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
129 pool->pages->slab = pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
130 } |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
131 |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
132 pool->last = pool->pages + pages; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
133 |
5634
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
134 pool->log_nomem = 1; |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
135 pool->log_ctx = &pool->zero; |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
136 pool->zero = '\0'; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
137 } |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
138 |
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
139 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
140 void * |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
141 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
142 { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
143 void *p; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
144 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
145 ngx_shmtx_lock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
146 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
147 p = ngx_slab_alloc_locked(pool, size); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
148 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
149 ngx_shmtx_unlock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
150 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
151 return p; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
152 } |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
153 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
154 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
155 void * |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
156 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
157 { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
158 size_t s; |
881
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
159 uintptr_t p, n, m, mask, *bitmap; |
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
160 ngx_uint_t i, slot, shift, map; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
161 ngx_slab_page_t *page, *prev, *slots; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
162 |
5766
abdb027be9d5
Core: bugfix for the ngx_slab_max_size case.
Jianjun Zheng <codeeply@gmail.com>
parents:
5726
diff
changeset
|
163 if (size > ngx_slab_max_size) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
164 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
165 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
967 | 166 "slab alloc: %uz", size); |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
167 |
4829
40de49cf6b37
Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents:
4764
diff
changeset
|
168 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift) |
40de49cf6b37
Fixed overflow if ngx_slab_alloc() is called with very big "size" argument.
Ruslan Ermilov <ru@nginx.com>
parents:
4764
diff
changeset
|
169 + ((size % ngx_pagesize) ? 1 : 0)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
170 if (page) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
171 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
172 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
173 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
174 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
175 p = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
176 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
177 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
178 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
179 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
180 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
181 if (size > pool->min_size) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
182 shift = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
183 for (s = size - 1; s >>= 1; shift++) { /* void */ } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
184 slot = shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
185 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
186 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
187 size = pool->min_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
188 shift = pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
189 slot = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
190 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
191 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
192 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
193 "slab alloc: %uz slot: %ui", size, slot); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
194 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
195 slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
196 page = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
197 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
198 if (page->next != page) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
199 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
200 if (shift < ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
201 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
202 do { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
203 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
204 bitmap = (uintptr_t *) (pool->start + p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
205 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
206 map = (1 << (ngx_pagesize_shift - shift)) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
207 / (sizeof(uintptr_t) * 8); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
208 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
209 for (n = 0; n < map; n++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
210 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
211 if (bitmap[n] != NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
212 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
213 for (m = 1, i = 0; m; m <<= 1, i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
214 if ((bitmap[n] & m)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
215 continue; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
216 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
217 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
218 bitmap[n] |= m; |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
219 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
220 i = ((n * sizeof(uintptr_t) * 8) << shift) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
221 + (i << shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
222 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
223 if (bitmap[n] == NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
224 for (n = n + 1; n < map; n++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
225 if (bitmap[n] != NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
226 p = (uintptr_t) bitmap + i; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
227 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
228 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
229 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
230 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
231 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
232 prev = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
233 (page->prev & ~NGX_SLAB_PAGE_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
234 prev->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
235 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
236 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
237 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
238 page->prev = NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
239 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
240 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
241 p = (uintptr_t) bitmap + i; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
242 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
243 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
244 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
245 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
246 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
247 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
248 page = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
249 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
250 } while (page); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
251 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
252 } else if (shift == ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
253 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
254 do { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
255 if (page->slab != NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
256 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
257 for (m = 1, i = 0; m; m <<= 1, i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
258 if ((page->slab & m)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
259 continue; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
260 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
261 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
262 page->slab |= m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
263 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
264 if (page->slab == NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
265 prev = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
266 (page->prev & ~NGX_SLAB_PAGE_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
267 prev->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
268 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
269 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
270 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
271 page->prev = NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
272 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
273 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
274 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
275 p += i << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
276 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
277 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
278 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
279 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
280 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
281 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
282 page = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
283 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
284 } while (page); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
285 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
286 } else { /* shift > ngx_slab_exact_shift */ |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
287 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
288 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
289 n = 1 << n; |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
290 n = ((uintptr_t) 1 << n) - 1; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
291 mask = n << NGX_SLAB_MAP_SHIFT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
292 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
293 do { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
294 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
295 |
881
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
296 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
297 m & mask; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
298 m <<= 1, i++) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
299 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
300 if ((page->slab & m)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
301 continue; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
302 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
303 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
304 page->slab |= m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
305 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
306 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
307 prev = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
308 (page->prev & ~NGX_SLAB_PAGE_MASK); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
309 prev->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
310 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
311 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
312 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
313 page->prev = NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
314 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
315 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
316 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
317 p += i << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
318 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
319 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
320 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
321 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
322 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
323 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
324 page = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
325 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
326 } while (page); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
327 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
328 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
329 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
330 page = ngx_slab_alloc_pages(pool, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
331 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
332 if (page) { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
333 if (shift < ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
334 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
335 bitmap = (uintptr_t *) (pool->start + p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
336 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
337 s = 1 << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
338 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
339 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
340 if (n == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
341 n = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
342 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
343 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
344 bitmap[0] = (2 << n) - 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
345 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
346 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
347 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
348 for (i = 1; i < map; i++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
349 bitmap[i] = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
350 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
351 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
352 page->slab = shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
353 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
354 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
355 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
356 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
357 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
358 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
359 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
360 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
361 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
362 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
363 } else if (shift == ngx_slab_exact_shift) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
364 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
365 page->slab = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
366 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
367 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
368 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
369 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
370 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
371 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
372 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
373 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
374 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
375 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
376 } else { /* shift > ngx_slab_exact_shift */ |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
377 |
881
5ed08a4c3bd3
fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents:
860
diff
changeset
|
378 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
379 page->next = &slots[slot]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
380 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
381 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
382 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
383 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
384 p = (page - pool->pages) << ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
385 p += (uintptr_t) pool->start; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
386 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
387 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
388 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
389 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
390 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
391 p = 0; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
392 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
393 done: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
394 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
395 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab alloc: %p", p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
396 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
397 return (void *) p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
398 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
399 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
400 |
5726
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
401 void * |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
402 ngx_slab_calloc(ngx_slab_pool_t *pool, size_t size) |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
403 { |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
404 void *p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
405 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
406 ngx_shmtx_lock(&pool->mutex); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
407 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
408 p = ngx_slab_calloc_locked(pool, size); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
409 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
410 ngx_shmtx_unlock(&pool->mutex); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
411 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
412 return p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
413 } |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
414 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
415 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
416 void * |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
417 ngx_slab_calloc_locked(ngx_slab_pool_t *pool, size_t size) |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
418 { |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
419 void *p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
420 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
421 p = ngx_slab_alloc_locked(pool, size); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
422 if (p) { |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
423 ngx_memzero(p, size); |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
424 } |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
425 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
426 return p; |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
427 } |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
428 |
25ade23cf281
Core: added ngx_slab_calloc() and ngx_slab_calloc_locked().
Ruslan Ermilov <ru@nginx.com>
parents:
5718
diff
changeset
|
429 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
430 void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
431 ngx_slab_free(ngx_slab_pool_t *pool, void *p) |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
432 { |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
433 ngx_shmtx_lock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
434 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
435 ngx_slab_free_locked(pool, p); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
436 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
437 ngx_shmtx_unlock(&pool->mutex); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
438 } |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
439 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
440 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
441 void |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
442 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p) |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
443 { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
444 size_t size; |
1018
06306a20b8bb
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
1016
diff
changeset
|
445 uintptr_t slab, m, *bitmap; |
06306a20b8bb
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
1016
diff
changeset
|
446 ngx_uint_t n, type, slot, shift, map; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
447 ngx_slab_page_t *slots, *page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
448 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
449 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
450 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
451 if ((u_char *) p < pool->start || (u_char *) p > pool->end) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
452 ngx_slab_error(pool, NGX_LOG_ALERT, "ngx_slab_free(): outside of pool"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
453 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
454 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
455 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
456 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
457 page = &pool->pages[n]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
458 slab = page->slab; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
459 type = page->prev & NGX_SLAB_PAGE_MASK; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
460 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
461 switch (type) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
462 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
463 case NGX_SLAB_SMALL: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
464 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
465 shift = slab & NGX_SLAB_SHIFT_MASK; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
466 size = 1 << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
467 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
468 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
469 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
470 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
471 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
472 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift; |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
473 m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
474 n /= (sizeof(uintptr_t) * 8); |
5362
79b9101cecf4
Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4829
diff
changeset
|
475 bitmap = (uintptr_t *) |
79b9101cecf4
Handling of ngx_int_t != intptr_t case.
Maxim Dounin <mdounin@mdounin.ru>
parents:
4829
diff
changeset
|
476 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1)); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
477 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
478 if (bitmap[n] & m) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
479 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
480 if (page->next == NULL) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
481 slots = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
482 ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
483 slot = shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
484 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
485 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
486 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
487 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
488 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
489 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
490 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
491 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
492 bitmap[n] &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
493 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
494 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
495 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
496 if (n == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
497 n = 1; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
498 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
499 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
500 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) { |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
501 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
502 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
503 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
504 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
505 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
506 for (n = 1; n < map; n++) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
507 if (bitmap[n]) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
508 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
509 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
510 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
511 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
512 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
513 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
514 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
515 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
516 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
517 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
518 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
519 case NGX_SLAB_EXACT: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
520 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
521 m = (uintptr_t) 1 << |
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
522 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
523 size = ngx_slab_exact_size; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
524 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
525 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
526 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
527 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
528 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
529 if (slab & m) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
530 if (slab == NGX_SLAB_BUSY) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
531 slots = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
532 ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
533 slot = ngx_slab_exact_shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
534 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
535 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
536 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
537 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
538 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
539 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
540 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
541 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
542 page->slab &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
543 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
544 if (page->slab) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
545 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
546 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
547 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
548 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
549 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
550 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
551 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
552 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
553 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
554 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
555 case NGX_SLAB_BIG: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
556 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
557 shift = slab & NGX_SLAB_SHIFT_MASK; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
558 size = 1 << shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
559 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
560 if ((uintptr_t) p & (size - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
561 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
562 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
563 |
1016
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
564 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift) |
d0e8c81d3bb7
fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents:
967
diff
changeset
|
565 + NGX_SLAB_MAP_SHIFT); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
566 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
567 if (slab & m) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
568 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
569 if (page->next == NULL) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
570 slots = (ngx_slab_page_t *) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
571 ((u_char *) pool + sizeof(ngx_slab_pool_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
572 slot = shift - pool->min_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
573 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
574 page->next = slots[slot].next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
575 slots[slot].next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
576 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
577 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
578 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
579 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
580 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
581 page->slab &= ~m; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
582 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
583 if (page->slab & NGX_SLAB_MAP_MASK) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
584 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
585 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
586 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
587 ngx_slab_free_pages(pool, page, 1); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
588 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
589 goto done; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
590 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
591 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
592 goto chunk_already_free; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
593 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
594 case NGX_SLAB_PAGE: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
595 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
596 if ((uintptr_t) p & (ngx_pagesize - 1)) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
597 goto wrong_chunk; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
598 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
599 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
600 if (slab == NGX_SLAB_PAGE_FREE) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
601 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
602 "ngx_slab_free(): page is already free"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
603 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
604 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
605 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
606 if (slab == NGX_SLAB_PAGE_BUSY) { |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
607 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
608 "ngx_slab_free(): pointer to wrong page"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
609 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
610 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
611 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
612 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
613 size = slab & ~NGX_SLAB_PAGE_START; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
614 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
615 ngx_slab_free_pages(pool, &pool->pages[n], size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
616 |
2711
6b87e875e87a
fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents:
2611
diff
changeset
|
617 ngx_slab_junk(p, size << ngx_pagesize_shift); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
618 |
2711
6b87e875e87a
fix building by BCC without NGX_DEBUG_MALLOC
Igor Sysoev <igor@sysoev.ru>
parents:
2611
diff
changeset
|
619 return; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
620 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
621 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
622 /* not reached */ |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
623 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
624 return; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
625 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
626 done: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
627 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
628 ngx_slab_junk(p, size); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
629 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
630 return; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
631 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
632 wrong_chunk: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
633 |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
634 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
635 "ngx_slab_free(): pointer to wrong chunk"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
636 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
637 goto fail; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
638 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
639 chunk_already_free: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
640 |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
641 ngx_slab_error(pool, NGX_LOG_ALERT, |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
642 "ngx_slab_free(): chunk is already free"); |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
643 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
644 fail: |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
645 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
646 return; |
357
e260514b9ad4
nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff
changeset
|
647 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
648 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
649 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
650 static ngx_slab_page_t * |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
651 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
652 { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
653 ngx_slab_page_t *page, *p; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
654 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
655 for (page = pool->free.next; page != &pool->free; page = page->next) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
656 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
657 if (page->slab >= pages) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
658 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
659 if (page->slab > pages) { |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
660 page[page->slab - 1].prev = (uintptr_t) &page[pages]; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
661 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
662 page[pages].slab = page->slab - pages; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
663 page[pages].next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
664 page[pages].prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
665 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
666 p = (ngx_slab_page_t *) page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
667 p->next = &page[pages]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
668 page->next->prev = (uintptr_t) &page[pages]; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
669 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
670 } else { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
671 p = (ngx_slab_page_t *) page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
672 p->next = page->next; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
673 page->next->prev = page->prev; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
674 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
675 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
676 page->slab = pages | NGX_SLAB_PAGE_START; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
677 page->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
678 page->prev = NGX_SLAB_PAGE; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
679 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
680 if (--pages == 0) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
681 return page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
682 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
683 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
684 for (p = page + 1; pages; pages--) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
685 p->slab = NGX_SLAB_PAGE_BUSY; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
686 p->next = NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
687 p->prev = NGX_SLAB_PAGE; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
688 p++; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
689 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
690 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
691 return page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
692 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
693 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
694 |
5634
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
695 if (pool->log_nomem) { |
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
696 ngx_slab_error(pool, NGX_LOG_CRIT, |
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
697 "ngx_slab_alloc() failed: no memory"); |
5024d29354f1
Core: slab log_nomem flag.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5362
diff
changeset
|
698 } |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
699 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
700 return NULL; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
701 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
702 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
703 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
704 static void |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
705 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
706 ngx_uint_t pages) |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
707 { |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
708 ngx_uint_t type; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
709 ngx_slab_page_t *prev, *join; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
710 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
711 page->slab = pages--; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
712 |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
713 if (pages) { |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
714 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t)); |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
715 } |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
716 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
717 if (page->next) { |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
718 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK); |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
719 prev->next = page->next; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
720 page->next->prev = page->prev; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
721 } |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
722 |
5718
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
723 join = page + page->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
724 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
725 if (join < pool->last) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
726 type = join->prev & NGX_SLAB_PAGE_MASK; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
727 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
728 if (type == NGX_SLAB_PAGE) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
729 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
730 if (join->next != NULL) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
731 pages += join->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
732 page->slab += join->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
733 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
734 prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
735 prev->next = join->next; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
736 join->next->prev = join->prev; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
737 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
738 join->slab = NGX_SLAB_PAGE_FREE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
739 join->next = NULL; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
740 join->prev = NGX_SLAB_PAGE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
741 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
742 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
743 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
744 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
745 if (page > pool->pages) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
746 join = page - 1; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
747 type = join->prev & NGX_SLAB_PAGE_MASK; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
748 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
749 if (type == NGX_SLAB_PAGE) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
750 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
751 if (join->slab == NGX_SLAB_PAGE_FREE) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
752 join = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
753 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
754 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
755 if (join->next != NULL) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
756 pages += join->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
757 join->slab += page->slab; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
758 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
759 prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
760 prev->next = join->next; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
761 join->next->prev = join->prev; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
762 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
763 page->slab = NGX_SLAB_PAGE_FREE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
764 page->next = NULL; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
765 page->prev = NGX_SLAB_PAGE; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
766 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
767 page = join; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
768 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
769 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
770 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
771 |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
772 if (pages) { |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
773 page[pages].prev = (uintptr_t) page; |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
774 } |
c46657e391a3
Core: slab allocator free pages defragmentation.
Maxim Dounin <mdounin@mdounin.ru>
parents:
5634
diff
changeset
|
775 |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
776 page->prev = (uintptr_t) &pool->free; |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
777 page->next = pool->free.next; |
966
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
778 |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
779 page->next->prev = (uintptr_t) page; |
715336c243e1
many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents:
881
diff
changeset
|
780 |
860
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
781 pool->free.next = page; |
201d017ea470
slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents:
826
diff
changeset
|
782 } |
2611
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
783 |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
784 |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
785 static void |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
786 ngx_slab_error(ngx_slab_pool_t *pool, ngx_uint_t level, char *text) |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
787 { |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
788 ngx_log_error(level, ngx_cycle->log, 0, "%s%s", text, pool->log_ctx); |
2bce3f6416c6
improve ngx_slab_alloc() error logging
Igor Sysoev <igor@sysoev.ru>
parents:
2610
diff
changeset
|
789 } |