Mercurial > hg > nginx
comparison src/core/ngx_slab.c @ 6823:88c8c3d65184
Slab: improved code readability.
No functional changes.
author | Ruslan Ermilov <ru@nginx.com> |
---|---|
date | Wed, 07 Dec 2016 22:25:37 +0300 |
parents | 87d7e640b45d |
children | ea12328518dc |
comparison
equal
deleted
inserted
replaced
6822:c045b4926b2c | 6823:88c8c3d65184 |
---|---|
37 #define NGX_SLAB_MAP_SHIFT 32 | 37 #define NGX_SLAB_MAP_SHIFT 32 |
38 | 38 |
39 #define NGX_SLAB_BUSY 0xffffffffffffffff | 39 #define NGX_SLAB_BUSY 0xffffffffffffffff |
40 | 40 |
41 #endif | 41 #endif |
42 | |
43 | |
44 #define ngx_slab_slots(pool) \ | |
45 (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t)) | |
46 | |
47 #define ngx_slab_page_type(page) ((page)->prev & NGX_SLAB_PAGE_MASK) | |
48 | |
49 #define ngx_slab_page_prev(page) \ | |
50 (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK) | |
51 | |
52 #define ngx_slab_page_addr(pool, page) \ | |
53 ((((page) - (pool)->pages) << ngx_pagesize_shift) \ | |
54 + (uintptr_t) (pool)->start) | |
42 | 55 |
43 | 56 |
44 #if (NGX_DEBUG_MALLOC) | 57 #if (NGX_DEBUG_MALLOC) |
45 | 58 |
46 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size) | 59 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size) |
74 { | 87 { |
75 u_char *p; | 88 u_char *p; |
76 size_t size; | 89 size_t size; |
77 ngx_int_t m; | 90 ngx_int_t m; |
78 ngx_uint_t i, n, pages; | 91 ngx_uint_t i, n, pages; |
79 ngx_slab_page_t *slots; | 92 ngx_slab_page_t *slots, *page; |
80 | 93 |
81 /* STUB */ | 94 /* STUB */ |
82 if (ngx_slab_max_size == 0) { | 95 if (ngx_slab_max_size == 0) { |
83 ngx_slab_max_size = ngx_pagesize / 2; | 96 ngx_slab_max_size = ngx_pagesize / 2; |
84 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t)); | 97 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t)); |
88 } | 101 } |
89 /**/ | 102 /**/ |
90 | 103 |
91 pool->min_size = 1 << pool->min_shift; | 104 pool->min_size = 1 << pool->min_shift; |
92 | 105 |
93 p = (u_char *) pool + sizeof(ngx_slab_pool_t); | 106 slots = ngx_slab_slots(pool); |
107 | |
108 p = (u_char *) slots; | |
94 size = pool->end - p; | 109 size = pool->end - p; |
95 | 110 |
96 ngx_slab_junk(p, size); | 111 ngx_slab_junk(p, size); |
97 | 112 |
98 slots = (ngx_slab_page_t *) p; | |
99 n = ngx_pagesize_shift - pool->min_shift; | 113 n = ngx_pagesize_shift - pool->min_shift; |
100 | 114 |
101 for (i = 0; i < n; i++) { | 115 for (i = 0; i < n; i++) { |
102 slots[i].slab = 0; | 116 slots[i].slab = 0; |
103 slots[i].next = &slots[i]; | 117 slots[i].next = &slots[i]; |
106 | 120 |
107 p += n * sizeof(ngx_slab_page_t); | 121 p += n * sizeof(ngx_slab_page_t); |
108 | 122 |
109 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t))); | 123 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t))); |
110 | 124 |
111 ngx_memzero(p, pages * sizeof(ngx_slab_page_t)); | |
112 | |
113 pool->pages = (ngx_slab_page_t *) p; | 125 pool->pages = (ngx_slab_page_t *) p; |
126 ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t)); | |
127 | |
128 page = pool->pages; | |
114 | 129 |
115 pool->free.prev = 0; | 130 pool->free.prev = 0; |
116 pool->free.next = (ngx_slab_page_t *) p; | 131 pool->free.next = page; |
117 | 132 |
118 pool->pages->slab = pages; | 133 page->slab = pages; |
119 pool->pages->next = &pool->free; | 134 page->next = &pool->free; |
120 pool->pages->prev = (uintptr_t) &pool->free; | 135 page->prev = (uintptr_t) &pool->free; |
121 | 136 |
122 pool->start = (u_char *) | 137 pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t), |
123 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t), | 138 ngx_pagesize); |
124 ngx_pagesize); | |
125 | 139 |
126 m = pages - (pool->end - pool->start) / ngx_pagesize; | 140 m = pages - (pool->end - pool->start) / ngx_pagesize; |
127 if (m > 0) { | 141 if (m > 0) { |
128 pages -= m; | 142 pages -= m; |
129 pool->pages->slab = pages; | 143 page->slab = pages; |
130 } | 144 } |
131 | 145 |
132 pool->last = pool->pages + pages; | 146 pool->last = pool->pages + pages; |
133 | 147 |
134 pool->log_nomem = 1; | 148 pool->log_nomem = 1; |
166 "slab alloc: %uz", size); | 180 "slab alloc: %uz", size); |
167 | 181 |
168 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift) | 182 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift) |
169 + ((size % ngx_pagesize) ? 1 : 0)); | 183 + ((size % ngx_pagesize) ? 1 : 0)); |
170 if (page) { | 184 if (page) { |
171 p = (page - pool->pages) << ngx_pagesize_shift; | 185 p = ngx_slab_page_addr(pool, page); |
172 p += (uintptr_t) pool->start; | |
173 | 186 |
174 } else { | 187 } else { |
175 p = 0; | 188 p = 0; |
176 } | 189 } |
177 | 190 |
189 } | 202 } |
190 | 203 |
191 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, | 204 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, |
192 "slab alloc: %uz slot: %ui", size, slot); | 205 "slab alloc: %uz slot: %ui", size, slot); |
193 | 206 |
194 slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t)); | 207 slots = ngx_slab_slots(pool); |
195 page = slots[slot].next; | 208 page = slots[slot].next; |
196 | 209 |
197 if (page->next != page) { | 210 if (page->next != page) { |
198 | 211 |
199 if (shift < ngx_slab_exact_shift) { | 212 if (shift < ngx_slab_exact_shift) { |
200 | 213 |
201 do { | 214 do { |
202 p = (page - pool->pages) << ngx_pagesize_shift; | 215 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page); |
203 bitmap = (uintptr_t *) (pool->start + p); | |
204 | 216 |
205 map = (1 << (ngx_pagesize_shift - shift)) | 217 map = (1 << (ngx_pagesize_shift - shift)) |
206 / (sizeof(uintptr_t) * 8); | 218 / (sizeof(uintptr_t) * 8); |
207 | 219 |
208 for (n = 0; n < map; n++) { | 220 for (n = 0; n < map; n++) { |
226 | 238 |
227 goto done; | 239 goto done; |
228 } | 240 } |
229 } | 241 } |
230 | 242 |
231 prev = (ngx_slab_page_t *) | 243 prev = ngx_slab_page_prev(page); |
232 (page->prev & ~NGX_SLAB_PAGE_MASK); | |
233 prev->next = page->next; | 244 prev->next = page->next; |
234 page->next->prev = page->prev; | 245 page->next->prev = page->prev; |
235 | 246 |
236 page->next = NULL; | 247 page->next = NULL; |
237 page->prev = NGX_SLAB_SMALL; | 248 page->prev = NGX_SLAB_SMALL; |
259 } | 270 } |
260 | 271 |
261 page->slab |= m; | 272 page->slab |= m; |
262 | 273 |
263 if (page->slab == NGX_SLAB_BUSY) { | 274 if (page->slab == NGX_SLAB_BUSY) { |
264 prev = (ngx_slab_page_t *) | 275 prev = ngx_slab_page_prev(page); |
265 (page->prev & ~NGX_SLAB_PAGE_MASK); | |
266 prev->next = page->next; | 276 prev->next = page->next; |
267 page->next->prev = page->prev; | 277 page->next->prev = page->prev; |
268 | 278 |
269 page->next = NULL; | 279 page->next = NULL; |
270 page->prev = NGX_SLAB_EXACT; | 280 page->prev = NGX_SLAB_EXACT; |
271 } | 281 } |
272 | 282 |
273 p = (page - pool->pages) << ngx_pagesize_shift; | 283 p = ngx_slab_page_addr(pool, page) + (i << shift); |
274 p += i << shift; | |
275 p += (uintptr_t) pool->start; | |
276 | 284 |
277 goto done; | 285 goto done; |
278 } | 286 } |
279 } | 287 } |
280 | 288 |
301 } | 309 } |
302 | 310 |
303 page->slab |= m; | 311 page->slab |= m; |
304 | 312 |
305 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) { | 313 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) { |
306 prev = (ngx_slab_page_t *) | 314 prev = ngx_slab_page_prev(page); |
307 (page->prev & ~NGX_SLAB_PAGE_MASK); | |
308 prev->next = page->next; | 315 prev->next = page->next; |
309 page->next->prev = page->prev; | 316 page->next->prev = page->prev; |
310 | 317 |
311 page->next = NULL; | 318 page->next = NULL; |
312 page->prev = NGX_SLAB_BIG; | 319 page->prev = NGX_SLAB_BIG; |
313 } | 320 } |
314 | 321 |
315 p = (page - pool->pages) << ngx_pagesize_shift; | 322 p = ngx_slab_page_addr(pool, page) + (i << shift); |
316 p += i << shift; | |
317 p += (uintptr_t) pool->start; | |
318 | 323 |
319 goto done; | 324 goto done; |
320 } | 325 } |
321 } | 326 } |
322 | 327 |
328 | 333 |
329 page = ngx_slab_alloc_pages(pool, 1); | 334 page = ngx_slab_alloc_pages(pool, 1); |
330 | 335 |
331 if (page) { | 336 if (page) { |
332 if (shift < ngx_slab_exact_shift) { | 337 if (shift < ngx_slab_exact_shift) { |
333 p = (page - pool->pages) << ngx_pagesize_shift; | 338 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page); |
334 bitmap = (uintptr_t *) (pool->start + p); | |
335 | 339 |
336 s = 1 << shift; | 340 s = 1 << shift; |
337 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s; | 341 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s; |
338 | 342 |
339 if (n == 0) { | 343 if (n == 0) { |
352 page->next = &slots[slot]; | 356 page->next = &slots[slot]; |
353 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; | 357 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; |
354 | 358 |
355 slots[slot].next = page; | 359 slots[slot].next = page; |
356 | 360 |
357 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n; | 361 p = ngx_slab_page_addr(pool, page) + s * n; |
358 p += (uintptr_t) pool->start; | |
359 | 362 |
360 goto done; | 363 goto done; |
361 | 364 |
362 } else if (shift == ngx_slab_exact_shift) { | 365 } else if (shift == ngx_slab_exact_shift) { |
363 | 366 |
365 page->next = &slots[slot]; | 368 page->next = &slots[slot]; |
366 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; | 369 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; |
367 | 370 |
368 slots[slot].next = page; | 371 slots[slot].next = page; |
369 | 372 |
370 p = (page - pool->pages) << ngx_pagesize_shift; | 373 p = ngx_slab_page_addr(pool, page); |
371 p += (uintptr_t) pool->start; | |
372 | 374 |
373 goto done; | 375 goto done; |
374 | 376 |
375 } else { /* shift > ngx_slab_exact_shift */ | 377 } else { /* shift > ngx_slab_exact_shift */ |
376 | 378 |
378 page->next = &slots[slot]; | 380 page->next = &slots[slot]; |
379 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; | 381 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; |
380 | 382 |
381 slots[slot].next = page; | 383 slots[slot].next = page; |
382 | 384 |
383 p = (page - pool->pages) << ngx_pagesize_shift; | 385 p = ngx_slab_page_addr(pool, page); |
384 p += (uintptr_t) pool->start; | |
385 | 386 |
386 goto done; | 387 goto done; |
387 } | 388 } |
388 } | 389 } |
389 | 390 |
454 } | 455 } |
455 | 456 |
456 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; | 457 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; |
457 page = &pool->pages[n]; | 458 page = &pool->pages[n]; |
458 slab = page->slab; | 459 slab = page->slab; |
459 type = page->prev & NGX_SLAB_PAGE_MASK; | 460 type = ngx_slab_page_type(page); |
460 | 461 |
461 switch (type) { | 462 switch (type) { |
462 | 463 |
463 case NGX_SLAB_SMALL: | 464 case NGX_SLAB_SMALL: |
464 | 465 |
476 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1)); | 477 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1)); |
477 | 478 |
478 if (bitmap[n] & m) { | 479 if (bitmap[n] & m) { |
479 | 480 |
480 if (page->next == NULL) { | 481 if (page->next == NULL) { |
481 slots = (ngx_slab_page_t *) | 482 slots = ngx_slab_slots(pool); |
482 ((u_char *) pool + sizeof(ngx_slab_pool_t)); | |
483 slot = shift - pool->min_shift; | 483 slot = shift - pool->min_shift; |
484 | 484 |
485 page->next = slots[slot].next; | 485 page->next = slots[slot].next; |
486 slots[slot].next = page; | 486 slots[slot].next = page; |
487 | 487 |
526 goto wrong_chunk; | 526 goto wrong_chunk; |
527 } | 527 } |
528 | 528 |
529 if (slab & m) { | 529 if (slab & m) { |
530 if (slab == NGX_SLAB_BUSY) { | 530 if (slab == NGX_SLAB_BUSY) { |
531 slots = (ngx_slab_page_t *) | 531 slots = ngx_slab_slots(pool); |
532 ((u_char *) pool + sizeof(ngx_slab_pool_t)); | |
533 slot = ngx_slab_exact_shift - pool->min_shift; | 532 slot = ngx_slab_exact_shift - pool->min_shift; |
534 | 533 |
535 page->next = slots[slot].next; | 534 page->next = slots[slot].next; |
536 slots[slot].next = page; | 535 slots[slot].next = page; |
537 | 536 |
565 + NGX_SLAB_MAP_SHIFT); | 564 + NGX_SLAB_MAP_SHIFT); |
566 | 565 |
567 if (slab & m) { | 566 if (slab & m) { |
568 | 567 |
569 if (page->next == NULL) { | 568 if (page->next == NULL) { |
570 slots = (ngx_slab_page_t *) | 569 slots = ngx_slab_slots(pool); |
571 ((u_char *) pool + sizeof(ngx_slab_pool_t)); | |
572 slot = shift - pool->min_shift; | 570 slot = shift - pool->min_shift; |
573 | 571 |
574 page->next = slots[slot].next; | 572 page->next = slots[slot].next; |
575 slots[slot].next = page; | 573 slots[slot].next = page; |
576 | 574 |
703 | 701 |
704 static void | 702 static void |
705 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, | 703 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, |
706 ngx_uint_t pages) | 704 ngx_uint_t pages) |
707 { | 705 { |
708 ngx_uint_t type; | |
709 ngx_slab_page_t *prev, *join; | 706 ngx_slab_page_t *prev, *join; |
710 | 707 |
711 page->slab = pages--; | 708 page->slab = pages--; |
712 | 709 |
713 if (pages) { | 710 if (pages) { |
714 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t)); | 711 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t)); |
715 } | 712 } |
716 | 713 |
717 if (page->next) { | 714 if (page->next) { |
718 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK); | 715 prev = ngx_slab_page_prev(page); |
719 prev->next = page->next; | 716 prev->next = page->next; |
720 page->next->prev = page->prev; | 717 page->next->prev = page->prev; |
721 } | 718 } |
722 | 719 |
723 join = page + page->slab; | 720 join = page + page->slab; |
724 | 721 |
725 if (join < pool->last) { | 722 if (join < pool->last) { |
726 type = join->prev & NGX_SLAB_PAGE_MASK; | 723 |
727 | 724 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) { |
728 if (type == NGX_SLAB_PAGE) { | |
729 | 725 |
730 if (join->next != NULL) { | 726 if (join->next != NULL) { |
731 pages += join->slab; | 727 pages += join->slab; |
732 page->slab += join->slab; | 728 page->slab += join->slab; |
733 | 729 |
734 prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); | 730 prev = ngx_slab_page_prev(join); |
735 prev->next = join->next; | 731 prev->next = join->next; |
736 join->next->prev = join->prev; | 732 join->next->prev = join->prev; |
737 | 733 |
738 join->slab = NGX_SLAB_PAGE_FREE; | 734 join->slab = NGX_SLAB_PAGE_FREE; |
739 join->next = NULL; | 735 join->next = NULL; |
742 } | 738 } |
743 } | 739 } |
744 | 740 |
745 if (page > pool->pages) { | 741 if (page > pool->pages) { |
746 join = page - 1; | 742 join = page - 1; |
747 type = join->prev & NGX_SLAB_PAGE_MASK; | 743 |
748 | 744 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) { |
749 if (type == NGX_SLAB_PAGE) { | |
750 | 745 |
751 if (join->slab == NGX_SLAB_PAGE_FREE) { | 746 if (join->slab == NGX_SLAB_PAGE_FREE) { |
752 join = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); | 747 join = ngx_slab_page_prev(join); |
753 } | 748 } |
754 | 749 |
755 if (join->next != NULL) { | 750 if (join->next != NULL) { |
756 pages += join->slab; | 751 pages += join->slab; |
757 join->slab += page->slab; | 752 join->slab += page->slab; |
758 | 753 |
759 prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); | 754 prev = ngx_slab_page_prev(join); |
760 prev->next = join->next; | 755 prev->next = join->next; |
761 join->next->prev = join->prev; | 756 join->next->prev = join->prev; |
762 | 757 |
763 page->slab = NGX_SLAB_PAGE_FREE; | 758 page->slab = NGX_SLAB_PAGE_FREE; |
764 page->next = NULL; | 759 page->next = NULL; |