Mercurial > hg > nginx

comparison src/core/ngx_slab.c @ 6823:88c8c3d65184

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Slab: improved code readability. No functional changes.
author Ruslan Ermilov <ru@nginx.com>
date Wed, 07 Dec 2016 22:25:37 +0300
parents 87d7e640b45d
children ea12328518dc
comparison
equal deleted inserted replaced
6822:c045b4926b2c 6823:88c8c3d65184
37 #define NGX_SLAB_MAP_SHIFT 32 37 #define NGX_SLAB_MAP_SHIFT 32
38 38
39 #define NGX_SLAB_BUSY 0xffffffffffffffff 39 #define NGX_SLAB_BUSY 0xffffffffffffffff
40 40
41 #endif 41 #endif
42
43
44 #define ngx_slab_slots(pool) \
45 (ngx_slab_page_t *) ((u_char *) (pool) + sizeof(ngx_slab_pool_t))
46
47 #define ngx_slab_page_type(page) ((page)->prev & NGX_SLAB_PAGE_MASK)
48
49 #define ngx_slab_page_prev(page) \
50 (ngx_slab_page_t *) ((page)->prev & ~NGX_SLAB_PAGE_MASK)
51
52 #define ngx_slab_page_addr(pool, page) \
53 ((((page) - (pool)->pages) << ngx_pagesize_shift) \
54 + (uintptr_t) (pool)->start)
42 55
43 56
44 #if (NGX_DEBUG_MALLOC) 57 #if (NGX_DEBUG_MALLOC)
45 58
46 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size) 59 #define ngx_slab_junk(p, size) ngx_memset(p, 0xA5, size)
74 { 87 {
75 u_char *p; 88 u_char *p;
76 size_t size; 89 size_t size;
77 ngx_int_t m; 90 ngx_int_t m;
78 ngx_uint_t i, n, pages; 91 ngx_uint_t i, n, pages;
79 ngx_slab_page_t *slots; 92 ngx_slab_page_t *slots, *page;
80 93
81 /* STUB */ 94 /* STUB */
82 if (ngx_slab_max_size == 0) { 95 if (ngx_slab_max_size == 0) {
83 ngx_slab_max_size = ngx_pagesize / 2; 96 ngx_slab_max_size = ngx_pagesize / 2;
84 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t)); 97 ngx_slab_exact_size = ngx_pagesize / (8 * sizeof(uintptr_t));
88 } 101 }
89 /**/ 102 /**/
90 103
91 pool->min_size = 1 << pool->min_shift; 104 pool->min_size = 1 << pool->min_shift;
92 105
93 p = (u_char *) pool + sizeof(ngx_slab_pool_t); 106 slots = ngx_slab_slots(pool);
107
108 p = (u_char *) slots;
94 size = pool->end - p; 109 size = pool->end - p;
95 110
96 ngx_slab_junk(p, size); 111 ngx_slab_junk(p, size);
97 112
98 slots = (ngx_slab_page_t *) p;
99 n = ngx_pagesize_shift - pool->min_shift; 113 n = ngx_pagesize_shift - pool->min_shift;
100 114
101 for (i = 0; i < n; i++) { 115 for (i = 0; i < n; i++) {
102 slots[i].slab = 0; 116 slots[i].slab = 0;
103 slots[i].next = &slots[i]; 117 slots[i].next = &slots[i];
106 120
107 p += n * sizeof(ngx_slab_page_t); 121 p += n * sizeof(ngx_slab_page_t);
108 122
109 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t))); 123 pages = (ngx_uint_t) (size / (ngx_pagesize + sizeof(ngx_slab_page_t)));
110 124
111 ngx_memzero(p, pages * sizeof(ngx_slab_page_t));
112
113 pool->pages = (ngx_slab_page_t *) p; 125 pool->pages = (ngx_slab_page_t *) p;
126 ngx_memzero(pool->pages, pages * sizeof(ngx_slab_page_t));
127
128 page = pool->pages;
114 129
115 pool->free.prev = 0; 130 pool->free.prev = 0;
116 pool->free.next = (ngx_slab_page_t *) p; 131 pool->free.next = page;
117 132
118 pool->pages->slab = pages; 133 page->slab = pages;
119 pool->pages->next = &pool->free; 134 page->next = &pool->free;
120 pool->pages->prev = (uintptr_t) &pool->free; 135 page->prev = (uintptr_t) &pool->free;
121 136
122 pool->start = (u_char *) 137 pool->start = ngx_align_ptr(p + pages * sizeof(ngx_slab_page_t),
123 ngx_align_ptr((uintptr_t) p + pages * sizeof(ngx_slab_page_t), 138 ngx_pagesize);
124 ngx_pagesize);
125 139
126 m = pages - (pool->end - pool->start) / ngx_pagesize; 140 m = pages - (pool->end - pool->start) / ngx_pagesize;
127 if (m > 0) { 141 if (m > 0) {
128 pages -= m; 142 pages -= m;
129 pool->pages->slab = pages; 143 page->slab = pages;
130 } 144 }
131 145
132 pool->last = pool->pages + pages; 146 pool->last = pool->pages + pages;
133 147
134 pool->log_nomem = 1; 148 pool->log_nomem = 1;
166 "slab alloc: %uz", size); 180 "slab alloc: %uz", size);
167 181
168 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift) 182 page = ngx_slab_alloc_pages(pool, (size >> ngx_pagesize_shift)
169 + ((size % ngx_pagesize) ? 1 : 0)); 183 + ((size % ngx_pagesize) ? 1 : 0));
170 if (page) { 184 if (page) {
171 p = (page - pool->pages) << ngx_pagesize_shift; 185 p = ngx_slab_page_addr(pool, page);
172 p += (uintptr_t) pool->start;
173 186
174 } else { 187 } else {
175 p = 0; 188 p = 0;
176 } 189 }
177 190
189 } 202 }
190 203
191 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0, 204 ngx_log_debug2(NGX_LOG_DEBUG_ALLOC, ngx_cycle->log, 0,
192 "slab alloc: %uz slot: %ui", size, slot); 205 "slab alloc: %uz slot: %ui", size, slot);
193 206
194 slots = (ngx_slab_page_t *) ((u_char *) pool + sizeof(ngx_slab_pool_t)); 207 slots = ngx_slab_slots(pool);
195 page = slots[slot].next; 208 page = slots[slot].next;
196 209
197 if (page->next != page) { 210 if (page->next != page) {
198 211
199 if (shift < ngx_slab_exact_shift) { 212 if (shift < ngx_slab_exact_shift) {
200 213
201 do { 214 do {
202 p = (page - pool->pages) << ngx_pagesize_shift; 215 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
203 bitmap = (uintptr_t *) (pool->start + p);
204 216
205 map = (1 << (ngx_pagesize_shift - shift)) 217 map = (1 << (ngx_pagesize_shift - shift))
206 / (sizeof(uintptr_t) * 8); 218 / (sizeof(uintptr_t) * 8);
207 219
208 for (n = 0; n < map; n++) { 220 for (n = 0; n < map; n++) {
226 238
227 goto done; 239 goto done;
228 } 240 }
229 } 241 }
230 242
231 prev = (ngx_slab_page_t *) 243 prev = ngx_slab_page_prev(page);
232 (page->prev & ~NGX_SLAB_PAGE_MASK);
233 prev->next = page->next; 244 prev->next = page->next;
234 page->next->prev = page->prev; 245 page->next->prev = page->prev;
235 246
236 page->next = NULL; 247 page->next = NULL;
237 page->prev = NGX_SLAB_SMALL; 248 page->prev = NGX_SLAB_SMALL;
259 } 270 }
260 271
261 page->slab |= m; 272 page->slab |= m;
262 273
263 if (page->slab == NGX_SLAB_BUSY) { 274 if (page->slab == NGX_SLAB_BUSY) {
264 prev = (ngx_slab_page_t *) 275 prev = ngx_slab_page_prev(page);
265 (page->prev & ~NGX_SLAB_PAGE_MASK);
266 prev->next = page->next; 276 prev->next = page->next;
267 page->next->prev = page->prev; 277 page->next->prev = page->prev;
268 278
269 page->next = NULL; 279 page->next = NULL;
270 page->prev = NGX_SLAB_EXACT; 280 page->prev = NGX_SLAB_EXACT;
271 } 281 }
272 282
273 p = (page - pool->pages) << ngx_pagesize_shift; 283 p = ngx_slab_page_addr(pool, page) + (i << shift);
274 p += i << shift;
275 p += (uintptr_t) pool->start;
276 284
277 goto done; 285 goto done;
278 } 286 }
279 } 287 }
280 288
301 } 309 }
302 310
303 page->slab |= m; 311 page->slab |= m;
304 312
305 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) { 313 if ((page->slab & NGX_SLAB_MAP_MASK) == mask) {
306 prev = (ngx_slab_page_t *) 314 prev = ngx_slab_page_prev(page);
307 (page->prev & ~NGX_SLAB_PAGE_MASK);
308 prev->next = page->next; 315 prev->next = page->next;
309 page->next->prev = page->prev; 316 page->next->prev = page->prev;
310 317
311 page->next = NULL; 318 page->next = NULL;
312 page->prev = NGX_SLAB_BIG; 319 page->prev = NGX_SLAB_BIG;
313 } 320 }
314 321
315 p = (page - pool->pages) << ngx_pagesize_shift; 322 p = ngx_slab_page_addr(pool, page) + (i << shift);
316 p += i << shift;
317 p += (uintptr_t) pool->start;
318 323
319 goto done; 324 goto done;
320 } 325 }
321 } 326 }
322 327
328 333
329 page = ngx_slab_alloc_pages(pool, 1); 334 page = ngx_slab_alloc_pages(pool, 1);
330 335
331 if (page) { 336 if (page) {
332 if (shift < ngx_slab_exact_shift) { 337 if (shift < ngx_slab_exact_shift) {
333 p = (page - pool->pages) << ngx_pagesize_shift; 338 bitmap = (uintptr_t *) ngx_slab_page_addr(pool, page);
334 bitmap = (uintptr_t *) (pool->start + p);
335 339
336 s = 1 << shift; 340 s = 1 << shift;
337 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s; 341 n = (1 << (ngx_pagesize_shift - shift)) / 8 / s;
338 342
339 if (n == 0) { 343 if (n == 0) {
352 page->next = &slots[slot]; 356 page->next = &slots[slot];
353 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL; 357 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_SMALL;
354 358
355 slots[slot].next = page; 359 slots[slot].next = page;
356 360
357 p = ((page - pool->pages) << ngx_pagesize_shift) + s * n; 361 p = ngx_slab_page_addr(pool, page) + s * n;
358 p += (uintptr_t) pool->start;
359 362
360 goto done; 363 goto done;
361 364
362 } else if (shift == ngx_slab_exact_shift) { 365 } else if (shift == ngx_slab_exact_shift) {
363 366
365 page->next = &slots[slot]; 368 page->next = &slots[slot];
366 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT; 369 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_EXACT;
367 370
368 slots[slot].next = page; 371 slots[slot].next = page;
369 372
370 p = (page - pool->pages) << ngx_pagesize_shift; 373 p = ngx_slab_page_addr(pool, page);
371 p += (uintptr_t) pool->start;
372 374
373 goto done; 375 goto done;
374 376
375 } else { /* shift > ngx_slab_exact_shift */ 377 } else { /* shift > ngx_slab_exact_shift */
376 378
378 page->next = &slots[slot]; 380 page->next = &slots[slot];
379 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG; 381 page->prev = (uintptr_t) &slots[slot] | NGX_SLAB_BIG;
380 382
381 slots[slot].next = page; 383 slots[slot].next = page;
382 384
383 p = (page - pool->pages) << ngx_pagesize_shift; 385 p = ngx_slab_page_addr(pool, page);
384 p += (uintptr_t) pool->start;
385 386
386 goto done; 387 goto done;
387 } 388 }
388 } 389 }
389 390
454 } 455 }
455 456
456 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift; 457 n = ((u_char *) p - pool->start) >> ngx_pagesize_shift;
457 page = &pool->pages[n]; 458 page = &pool->pages[n];
458 slab = page->slab; 459 slab = page->slab;
459 type = page->prev & NGX_SLAB_PAGE_MASK; 460 type = ngx_slab_page_type(page);
460 461
461 switch (type) { 462 switch (type) {
462 463
463 case NGX_SLAB_SMALL: 464 case NGX_SLAB_SMALL:
464 465
476 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1)); 477 ((uintptr_t) p & ~((uintptr_t) ngx_pagesize - 1));
477 478
478 if (bitmap[n] & m) { 479 if (bitmap[n] & m) {
479 480
480 if (page->next == NULL) { 481 if (page->next == NULL) {
481 slots = (ngx_slab_page_t *) 482 slots = ngx_slab_slots(pool);
482 ((u_char *) pool + sizeof(ngx_slab_pool_t));
483 slot = shift - pool->min_shift; 483 slot = shift - pool->min_shift;
484 484
485 page->next = slots[slot].next; 485 page->next = slots[slot].next;
486 slots[slot].next = page; 486 slots[slot].next = page;
487 487
526 goto wrong_chunk; 526 goto wrong_chunk;
527 } 527 }
528 528
529 if (slab & m) { 529 if (slab & m) {
530 if (slab == NGX_SLAB_BUSY) { 530 if (slab == NGX_SLAB_BUSY) {
531 slots = (ngx_slab_page_t *) 531 slots = ngx_slab_slots(pool);
532 ((u_char *) pool + sizeof(ngx_slab_pool_t));
533 slot = ngx_slab_exact_shift - pool->min_shift; 532 slot = ngx_slab_exact_shift - pool->min_shift;
534 533
535 page->next = slots[slot].next; 534 page->next = slots[slot].next;
536 slots[slot].next = page; 535 slots[slot].next = page;
537 536
565 + NGX_SLAB_MAP_SHIFT); 564 + NGX_SLAB_MAP_SHIFT);
566 565
567 if (slab & m) { 566 if (slab & m) {
568 567
569 if (page->next == NULL) { 568 if (page->next == NULL) {
570 slots = (ngx_slab_page_t *) 569 slots = ngx_slab_slots(pool);
571 ((u_char *) pool + sizeof(ngx_slab_pool_t));
572 slot = shift - pool->min_shift; 570 slot = shift - pool->min_shift;
573 571
574 page->next = slots[slot].next; 572 page->next = slots[slot].next;
575 slots[slot].next = page; 573 slots[slot].next = page;
576 574
703 701
704 static void 702 static void
705 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page, 703 ngx_slab_free_pages(ngx_slab_pool_t *pool, ngx_slab_page_t *page,
706 ngx_uint_t pages) 704 ngx_uint_t pages)
707 { 705 {
708 ngx_uint_t type;
709 ngx_slab_page_t *prev, *join; 706 ngx_slab_page_t *prev, *join;
710 707
711 page->slab = pages--; 708 page->slab = pages--;
712 709
713 if (pages) { 710 if (pages) {
714 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t)); 711 ngx_memzero(&page[1], pages * sizeof(ngx_slab_page_t));
715 } 712 }
716 713
717 if (page->next) { 714 if (page->next) {
718 prev = (ngx_slab_page_t *) (page->prev & ~NGX_SLAB_PAGE_MASK); 715 prev = ngx_slab_page_prev(page);
719 prev->next = page->next; 716 prev->next = page->next;
720 page->next->prev = page->prev; 717 page->next->prev = page->prev;
721 } 718 }
722 719
723 join = page + page->slab; 720 join = page + page->slab;
724 721
725 if (join < pool->last) { 722 if (join < pool->last) {
726 type = join->prev & NGX_SLAB_PAGE_MASK; 723
727 724 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
728 if (type == NGX_SLAB_PAGE) {
729 725
730 if (join->next != NULL) { 726 if (join->next != NULL) {
731 pages += join->slab; 727 pages += join->slab;
732 page->slab += join->slab; 728 page->slab += join->slab;
733 729
734 prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); 730 prev = ngx_slab_page_prev(join);
735 prev->next = join->next; 731 prev->next = join->next;
736 join->next->prev = join->prev; 732 join->next->prev = join->prev;
737 733
738 join->slab = NGX_SLAB_PAGE_FREE; 734 join->slab = NGX_SLAB_PAGE_FREE;
739 join->next = NULL; 735 join->next = NULL;
742 } 738 }
743 } 739 }
744 740
745 if (page > pool->pages) { 741 if (page > pool->pages) {
746 join = page - 1; 742 join = page - 1;
747 type = join->prev & NGX_SLAB_PAGE_MASK; 743
748 744 if (ngx_slab_page_type(join) == NGX_SLAB_PAGE) {
749 if (type == NGX_SLAB_PAGE) {
750 745
751 if (join->slab == NGX_SLAB_PAGE_FREE) { 746 if (join->slab == NGX_SLAB_PAGE_FREE) {
752 join = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); 747 join = ngx_slab_page_prev(join);
753 } 748 }
754 749
755 if (join->next != NULL) { 750 if (join->next != NULL) {
756 pages += join->slab; 751 pages += join->slab;
757 join->slab += page->slab; 752 join->slab += page->slab;
758 753
759 prev = (ngx_slab_page_t *) (join->prev & ~NGX_SLAB_PAGE_MASK); 754 prev = ngx_slab_page_prev(join);
760 prev->next = join->next; 755 prev->next = join->next;
761 join->next->prev = join->prev; 756 join->next->prev = join->prev;
762 757
763 page->slab = NGX_SLAB_PAGE_FREE; 758 page->slab = NGX_SLAB_PAGE_FREE;
764 page->next = NULL; 759 page->next = NULL;