Mercurial > hg > nginx

annotate src/core/ngx_slab.c @ 1565:4c43e25d11ea

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix English grammar
author Igor Sysoev <igor@sysoev.ru>
date Sun, 14 Oct 2007 18:56:15 +0000
parents 06306a20b8bb
children 31622d9f2c0d
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
1
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
2 /*
444
42d11f017717 nginx-0.1.0-2004-09-29-20:00:49 import; remove years from copyright
Igor Sysoev <igor@sysoev.ru>
parents: 441
diff changeset
3 * Copyright (C) Igor Sysoev
441
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
4 */
da8c5707af39 nginx-0.1.0-2004-09-28-12:34:51 import; set copyright and remove unused files
Igor Sysoev <igor@sysoev.ru>
parents: 357
diff changeset
5
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
6 #include <ngx_config.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
7 #include <ngx_core.h>
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
8
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
9 /*
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
10
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
11 12
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
12 2048 2 11
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
13 1024 4 10
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
14 512 8 9
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
15 256 16 8
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
17 128 32 4 32 7
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
18
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
19 64 64 8 63 6 1
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
20 32 128 16 127 5 1
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
21 16 256 32 254 4 2
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
22 8 512 64 504 3 8
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
23
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
24 */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
25
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
26
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
27 #define NGX_SLAB_PAGE_MASK 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
28 #define NGX_SLAB_PAGE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
29 #define NGX_SLAB_BIG 1
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
30 #define NGX_SLAB_EXACT 2
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
31 #define NGX_SLAB_SMALL 3
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
32
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
33 #if (NGX_PTR_SIZE == 4)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
34
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
35 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
36 #define NGX_SLAB_PAGE_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
37 #define NGX_SLAB_PAGE_START 0x80000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
38
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
39 #define NGX_SLAB_SHIFT_MASK 0x0000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
40 #define NGX_SLAB_MAP_MASK 0xffff0000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
41 #define NGX_SLAB_MAP_SHIFT 16
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
42
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
43 #define NGX_SLAB_BUSY 0xffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
44
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
45 #else /* (NGX_PTR_SIZE == 8) */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
46
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
47 #define NGX_SLAB_PAGE_FREE 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
48 #define NGX_SLAB_PAGE_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
49 #define NGX_SLAB_PAGE_START 0x8000000000000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
50
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
51 #define NGX_SLAB_SHIFT_MASK 0x000000000000000f
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
52 #define NGX_SLAB_MAP_MASK 0xffffffff00000000
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
53 #define NGX_SLAB_MAP_SHIFT 32
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
54
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
55 #define NGX_SLAB_BUSY 0xffffffffffffffff
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
56
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
57 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
58
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
59
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
60 #if (NGX_DEBUG_MALLOC)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
61 #define ngx_slab_junk(p, size) ngx_memset(p, 0xD0, size)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
62 #else
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
63 #define ngx_slab_junk(p, size)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
64 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
65
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
66 static ngx_slab_page_t *ngx_slab_alloc_pages(ngx_slab_pool_t *pool,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
67 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
68 static void ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
69 ngx_uint_t pages);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
70
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
71
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
72 static ngx_uint_t ngx_slab_max_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
73 static ngx_uint_t ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
74 static ngx_uint_t ngx_slab_exact_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
75
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
76
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
77 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
78 ngx_slab_init(ngx_slab_pool_t *pool)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
79 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
80 u_char *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
81 size_t size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
82 ngx_int_t m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
83 ngx_uint_t i, n, pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
84 ngx_slab_page_t *slots;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
85
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
86 /* STUB */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
87 if (ngx_slab_max_size == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
88 ngx_slab_max_size = ngx_pagesize / 2;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
89 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
90 for (n = ngx_slab_exact_size; n >>= 1; ngx_slab_exact_shift++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
91 /* void */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
92 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
93 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
94 /**/
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
95
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
96 pool->min_size = 1 << pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
97
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
98 p = (u_char *) pool + sizeof(ngx_slab_pool_t);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
99 size = pool->end - p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
100
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
101 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
102
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
103 slots = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
104 n = ngx_pagesize_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
105
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
106 for (i = 0; i < n; i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
107 slots[i].slab = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
108 slots[i].next = &slots[i];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
109 slots[i].prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
110 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
111
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
112 p += n * sizeof(ngx_slab_page_t);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
113
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
114 /* STUB: possible overflow on 64-bit platform */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
115 pages = (ngx_uint_t) ((uint64_t) size * ngx_pagesize
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
116 / (ngx_pagesize + sizeof(ngx_slab_page_t))
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
117 / ngx_pagesize);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
118
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
119 ngx_memzero(p, pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
120
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
121 pool->pages = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
122
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
123 pool->free.prev = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
124 pool->free.next = (ngx_slab_page_t *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
125
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
126 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
127 pool->pages->next = &pool->free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
128 pool->pages->prev = (uintptr_t) &pool->free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
129
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
130 pool->start = (u_char *)
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
131 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t),
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
132 ngx_pagesize);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
133
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
134 m = pages - (pool->end - pool->start) / ngx_pagesize;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
135 if (m > 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
136 pages -= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
137 pool->pages->slab = pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
138 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
139
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
140 #if 0
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
141 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "slab: %p, %p, %ui, %d",
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
142 pool, pool->start, pages,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
143 (pool->end - pool->start) / ngx_pagesize - pages);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
144 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
145 }
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
146
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
147
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
148 void *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
149 ngx_slab_alloc(ngx_slab_pool_t *pool, size_t size)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
150 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
151 void *p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
152
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
153 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
154
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
155 p = ngx_slab_alloc_locked(pool, size);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
156
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
157 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
158
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
159 return p;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
160 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
161
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
162
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
163 void *
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
164 ngx_slab_alloc_locked(ngx_slab_pool_t *pool, size_t size)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
165 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
166 size_t s;
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
167 uintptr_t p, n, m, mask, *bitmap;
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
168 ngx_uint_t i, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
169 ngx_slab_page_t *page, *prev, *slots;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
170
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
171 if (size >= ngx_slab_max_size) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
172
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
173 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
967
1cb178720355 style fix
Igor Sysoev <igor@sysoev.ru>
parents: 966
diff changeset
174 "slab alloc: %uz", size);
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
175
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
176 page = ngx_slab_alloc_pages(pool, (size + ngx_pagesize - 1)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
177 >> ngx_pagesize_shift);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
178 if (page) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
179 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
180 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
181
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
182 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
183 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
184 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
185
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
186 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
187 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
188
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
189 if (size > pool->min_size) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
190 shift = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
191 for (s = size - 1; s >>= 1; shift++) { /* void */ }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
192 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
193
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
194 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
195 size = pool->min_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
196 shift = pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
197 slot = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
198 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
199
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
200 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
201 "slab alloc: %uz slot: %ui", size, slot);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
202
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
203 slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
204 page = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
205
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
206 if (page->next != page) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
207
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
208 if (shift < ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
209
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
210 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
211 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
212 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
213
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
214 map = (1 << (ngx_pagesize_shift - shift))
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
215 / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
216
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
217 for (n = 0; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
218
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
219 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
220
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
221 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
222 if ((bitmap[n] & m)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
223 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
224 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
225
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
226 bitmap[n] |= m;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
227
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
228 i = ((n * sizeof(uintptr_t) * 8) << shift)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
229 + (i << shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
230
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
231 if (bitmap[n] == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
232 for (n = n + 1; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
233 if (bitmap[n] != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
234 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
235
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
236 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
237 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
238 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
239
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
240 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
241 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
242 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
243 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
244
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
245 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
246 page->prev = NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
247 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
248
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
249 p = (uintptr_t) bitmap + i;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
250
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
251 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
252 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
253 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
254 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
255
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
256 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
257
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
258 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
259
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
260 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
261
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
262 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
263 if (page->slab != NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
264
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
265 for (m = 1, i = 0; m; m <<= 1, i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
266 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
267 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
268 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
269
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
270 page->slab |= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
271
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
272 if (page->slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
273 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
274 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
275 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
276 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
277
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
278 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
279 page->prev = NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
280 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
281
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
282 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
283 p += i << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
284 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
285
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
286 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
287 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
288 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
289
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
290 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
291
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
292 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
293
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
294 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
295
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
296 n = ngx_pagesize_shift - (page->slab & NGX_SLAB_SHIFT_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
297 n = 1 << n;
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
298 n = ((uintptr_t) 1 << n) - 1;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
299 mask = n << NGX_SLAB_MAP_SHIFT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
300
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
301 do {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
302 if ((page->slab & NGX_SLAB_MAP_MASK) != mask) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
303
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
304 for (m = (uintptr_t) 1 << NGX_SLAB_MAP_SHIFT, i = 0;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
305 m & mask;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
306 m <<= 1, i++)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
307 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
308 if ((page->slab & m)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
309 continue;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
310 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
311
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
312 page->slab |= m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
313
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
314 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
315 prev = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
316 (page->prev & ~NGX_SLAB_PAGE_MASK);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
317 prev->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
318 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
319
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
320 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
321 page->prev = NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
322 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
323
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
324 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
325 p += i << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
326 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
327
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
328 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
329 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
330 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
331
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
332 page = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
333
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
334 } while (page);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
335 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
336 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
337
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
338 page = ngx_slab_alloc_pages(pool, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
339
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
340 if (page) {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
341 if (shift < ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
342 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
343 bitmap = (uintptr_t *) (pool->start + p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
344
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
345 s = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
346 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
347
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
348 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
349 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
350 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
351
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
352 bitmap[0] = (2 << n) - 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
353
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
354 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
355
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
356 for (i = 1; i < map; i++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
357 bitmap[i] = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
358 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
359
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
360 page->slab = shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
361 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
362 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
363
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
364 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
365
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
366 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
367 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
368
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
369 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
370
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
371 } else if (shift == ngx_slab_exact_shift) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
372
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
373 page->slab = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
374 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
375 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
376
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
377 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
378
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
379 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
380 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
381
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
382 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
383
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
384 } else { /* shift > ngx_slab_exact_shift */
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
385
881
5ed08a4c3bd3 fix building on 64-bit platforms
Igor Sysoev <igor@sysoev.ru>
parents: 860
diff changeset
386 page->slab = ((uintptr_t) 1 << NGX_SLAB_MAP_SHIFT) | shift;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
387 page->next = &slots[slot];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
388 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
389
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
390 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
391
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
392 p = (page - pool->pages) << ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
393 p += (uintptr_t) pool->start;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
394
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
395 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
396 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
397 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
398
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
399 p = 0;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
400
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
401 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
402
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
403 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab alloc: %p", p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
404
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
405 return (void *) p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
406 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
407
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
408
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
409 void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
410 ngx_slab_free(ngx_slab_pool_t *pool, void *p)
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
411 {
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
412 ngx_shmtx_lock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
413
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
414 ngx_slab_free_locked(pool, p);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
415
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
416 ngx_shmtx_unlock(&pool->mutex);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
417 }
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
418
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
419
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
420 void
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
421 ngx_slab_free_locked(ngx_slab_pool_t *pool, void *p)
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
422 {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
423 size_t size;
1018
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
424 uintptr_t slab, m, *bitmap;
06306a20b8bb fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 1016
diff changeset
425 ngx_uint_t n, type, slot, shift, map;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
426 ngx_slab_page_t *slots, *page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
427
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
428 ngx_log_debug1(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, "slab free: %p", p);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
429
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
430 if ((u_char *) p < pool->start || (u_char *) p > pool->end) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
431 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
432 "ngx_slab_free(): outside of pool");
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
433 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
434 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
435
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
436 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
437 page = &pool->pages[n];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
438 slab = page->slab;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
439 type = page->prev & NGX_SLAB_PAGE_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
440
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
441 switch (type) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
442
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
443 case NGX_SLAB_SMALL:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
444
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
445 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
446 size = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
447
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
448 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
449 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
450 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
451
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
452 n = ((uintptr_t) p & (ngx_pagesize - 1)) >> shift;
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
453 m = (uintptr_t) 1 << (n & (sizeof(uintptr_t) * 8 - 1));
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
454 n /= (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
455 bitmap = (uintptr_t *) ((uintptr_t) p & ~(ngx_pagesize - 1));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
456
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
457 if (bitmap[n] & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
458
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
459 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
460 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
461 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
462 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
463
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
464 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
465 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
466
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
467 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
468 page->next->prev = (uintptr_t) page | NGX_SLAB_SMALL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
469 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
470
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
471 bitmap[n] &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
472
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
473 n = (1 << (ngx_pagesize_shift - shift)) / 8 / (1 << shift);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
474
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
475 if (n == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
476 n = 1;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
477 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
478
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
479 if (bitmap[0] & ~(((uintptr_t) 1 << n) - 1)) {
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
480 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
481 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
482
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
483 map = (1 << (ngx_pagesize_shift - shift)) / (sizeof(uintptr_t) * 8);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
484
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
485 for (n = 1; n < map; n++) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
486 if (bitmap[n]) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
487 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
488 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
489 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
490
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
491 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
492
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
493 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
494 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
495
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
496 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
497
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
498 case NGX_SLAB_EXACT:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
499
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
500 m = (uintptr_t) 1 <<
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
501 (((uintptr_t) p & (ngx_pagesize - 1)) >> ngx_slab_exact_shift);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
502 size = ngx_slab_exact_size;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
503
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
504 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
505 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
506 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
507
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
508 if (slab & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
509 if (slab == NGX_SLAB_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
510 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
511 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
512 slot = ngx_slab_exact_shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
513
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
514 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
515 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
516
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
517 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
518 page->next->prev = (uintptr_t) page | NGX_SLAB_EXACT;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
519 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
520
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
521 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
522
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
523 if (page->slab) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
524 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
525 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
526
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
527 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
528
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
529 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
530 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
531
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
532 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
533
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
534 case NGX_SLAB_BIG:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
535
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
536 shift = slab & NGX_SLAB_SHIFT_MASK;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
537 size = 1 << shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
538
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
539 if ((uintptr_t) p & (size - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
540 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
541 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
542
1016
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
543 m = (uintptr_t) 1 << ((((uintptr_t) p & (ngx_pagesize - 1)) >> shift)
d0e8c81d3bb7 fix slab allocator on 64-bit platfroms
Igor Sysoev <igor@sysoev.ru>
parents: 967
diff changeset
544 + NGX_SLAB_MAP_SHIFT);
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
545
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
546 if (slab & m) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
547
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
548 if (page->next == NULL) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
549 slots = (ngx_slab_page_t *)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
550 ((u_char *) pool + sizeof(ngx_slab_pool_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
551 slot = shift - pool->min_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
552
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
553 page->next = slots[slot].next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
554 slots[slot].next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
555
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
556 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
557 page->next->prev = (uintptr_t) page | NGX_SLAB_BIG;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
558 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
559
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
560 page->slab &= ~m;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
561
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
562 if (page->slab & NGX_SLAB_MAP_MASK) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
563 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
564 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
565
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
566 ngx_slab_free_pages(pool, page, 1);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
567
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
568 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
569 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
570
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
571 goto chunk_already_free;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
572
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
573 case NGX_SLAB_PAGE:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
574
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
575 if ((uintptr_t) p & (ngx_pagesize - 1)) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
576 goto wrong_chunk;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
577 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
578
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
579 if (slab == NGX_SLAB_PAGE_FREE) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
580 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
581 "ngx_slab_free(): page is already free");
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
582 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
583 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
584
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
585 if (slab == NGX_SLAB_PAGE_BUSY) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
586 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
587 "ngx_slab_free(): pointer to wrong page");
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
588 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
589 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
590
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
591 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
592 size = slab & ~NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
593
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
594 ngx_slab_free_pages(pool, &pool->pages[n], size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
595
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
596 size <<= ngx_pagesize_shift;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
597
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
598 goto done;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
599 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
600
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
601 /* not reached */
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
602
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
603 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
604
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
605 done:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
606
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
607 ngx_slab_junk(p, size);
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
608
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
609 return;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
610
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
611 wrong_chunk:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
612
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
613 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
614 "ngx_slab_free(): pointer to wrong chunk");
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
615
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
616 goto fail;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
617
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
618 chunk_already_free:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
619
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
620 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
621 "ngx_slab_free(): chunk is already free");
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
622
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
623 fail:
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
624
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
625 return;
357
e260514b9ad4 nginx-0.0.7-2004-06-16-23:36:07 import
Igor Sysoev <igor@sysoev.ru>
parents:
diff changeset
626 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
627
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
628
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
629 static ngx_slab_page_t *
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
630 ngx_slab_alloc_pages(ngx_slab_pool_t *pool, ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
631 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
632 ngx_slab_page_t *page, *p;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
633
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
634 for (page = pool->free.next; page != &pool->free; page = page->next) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
635
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
636 if (page->slab >= pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
637
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
638 if (page->slab > pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
639 page[pages].slab = page->slab - pages;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
640 page[pages].next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
641 page[pages].prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
642
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
643 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
644 p->next = &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
645 page->next->prev = (uintptr_t) &page[pages];
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
646
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
647 } else {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
648 p = (ngx_slab_page_t *) page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
649 p->next = page->next;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
650 page->next->prev = page->prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
651 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
652
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
653 page->slab = pages | NGX_SLAB_PAGE_START;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
654
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
655 #if (NGX_DEBUG)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
656 page->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
657 page->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
658 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
659
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
660 if (--pages == 0) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
661 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
662 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
663
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
664 for (p = page + 1; pages; pages--) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
665 p->slab = NGX_SLAB_PAGE_BUSY;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
666 #if (NGX_DEBUG)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
667 p->next = NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
668 p->prev = NGX_SLAB_PAGE;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
669 #endif
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
670 p++;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
671 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
672
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
673 return page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
674 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
675 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
676
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
677 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, NGX_ENOMEM,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
678 "ngx_slab_alloc(): failed");
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
679
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
680 return NULL;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
681 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
682
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
683
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
684 static void
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
685 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
686 ngx_uint_t pages)
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
687 {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
688 ngx_slab_page_t *prev;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
689
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
690 page->slab = pages--;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
691
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
692 if (pages) {
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
693 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
694 }
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
695
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
696 if (page->next) {
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
697 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK);
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
698 prev->next = page->next;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
699 page->next->prev = page->prev;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
700 }
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
701
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
702 page->prev = (uintptr_t) &pool->free;
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
703 page->next = pool->free.next;
966
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
704
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
705 page->next->prev = (uintptr_t) page;
715336c243e1 many bug fixes and ngx_slab_alloc_locked()/ngx_slab_free_locked()
Igor Sysoev <igor@sysoev.ru>
parents: 881
diff changeset
706
860
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
707 pool->free.next = page;
201d017ea470 slab allocator in shared memory
Igor Sysoev <igor@sysoev.ru>
parents: 826
diff changeset
708 }