path: root/src/nxt_mem_zone.c

/*
 * Copyright (C) Igor Sysoev
 * Copyright (C) NGINX, Inc.
 */

#include <nxt_main.h>


#define NXT_MEM_ZONE_PAGE_FREE      0
/*
 * A page was never allocated before so it should be filled with
 * junk on the first time allocation if memory debugging is enabled.
 */
#define NXT_MEM_ZONE_PAGE_FRESH     1

/* An entire page is currently used, no chunks inside the page. */
#define NXT_MEM_ZONE_PAGE_USED      2


typedef struct nxt_mem_zone_page_s  nxt_mem_zone_page_t;

struct nxt_mem_zone_page_s {
    /*
     * A size of page chunks if value is greater than or equal to 16.
     * Otherwise it is used to mark page state: NXT_MEM_ZONE_PAGE_FREE,
     * NXT_MEM_ZONE_PAGE_FRESH, and NXT_MEM_ZONE_PAGE_USED.
     */
    uint16_t               size;

    /* A number of free chunks of a chunked page. */
    uint16_t               chunks;

    union {
        /* A chunk bitmap if a number of chunks is lesser than 32. */
        uint8_t            map[4];
        /*
         * The count is a number of successive occupied pages in the first
         * page.  In the next occupied pages and in all free pages the count
         * is zero, because a number of successive free pages is stored in
         * free block size resided in beginning of the first free page.
         */
        uint32_t           count;
    } u;

    /* Used for slot list of pages with free chunks. */
    nxt_mem_zone_page_t    *next;

    /*
     * Used to link of all pages including free, chunked and occupied
     * pages to coalesce free pages.
     */
    nxt_queue_link_t       link;
};


typedef struct {
    uint32_t               size;
    uint32_t               chunks;
    uint32_t               start;
    uint32_t               map_size;
    nxt_mem_zone_page_t    *pages;
} nxt_mem_zone_slot_t;


typedef struct {
    NXT_RBTREE_NODE        (node);
    uint32_t               size;
} nxt_mem_zone_free_block_t;


struct nxt_mem_zone_s {
    nxt_thread_spinlock_t  lock;
    nxt_mem_zone_page_t    *pages;
    nxt_mem_zone_page_t    sentinel_page;
    nxt_rbtree_t           free_pages;

    uint32_t               page_size_shift;
    uint32_t               page_size_mask;
    uint32_t               max_chunk_size;
    uint32_t               small_bitmap_min_size;

    u_char                 *start;
    u_char                 *end;

    nxt_mem_zone_slot_t    slots[];
};


#define                                                                       \
nxt_mem_zone_page_addr(zone, page)                                            \
    (void *) (zone->start + ((page - zone->pages) << zone->page_size_shift))


#define                                                                       \
nxt_mem_zone_addr_page(zone, addr)                                            \
    &zone->pages[((u_char *) addr - zone->start) >> zone->page_size_shift]


#define                                                                       \
nxt_mem_zone_page_is_free(page)                                               \
    (page->size < NXT_MEM_ZONE_PAGE_USED)


#define                                                                       \
nxt_mem_zone_page_is_chunked(page)                                            \
    (page->size >= 16)


#define                                                                       \
nxt_mem_zone_page_bitmap(zone, slot)                                          \
    (slot->size < zone->small_bitmap_min_size)


#define                                                                       \
nxt_mem_zone_set_chunk_free(map, chunk)                                       \
    map[chunk / 8] &= ~(0x80 >> (chunk & 7))


#define                                                                       \
nxt_mem_zone_chunk_is_free(map, chunk)                                        \
    ((map[chunk / 8] & (0x80 >> (chunk & 7))) == 0)


#define                                                                       \
nxt_mem_zone_fresh_junk(p, size)                                              \
    nxt_memset((p), 0xA5, size)


#define                                                                       \
nxt_mem_zone_free_junk(p, size)                                               \
    nxt_memset((p), 0x5A, size)


static uint32_t nxt_mem_zone_pages(u_char *start, size_t zone_size,
    nxt_uint_t page_size);
static void *nxt_mem_zone_slots_init(nxt_mem_zone_t *zone,
    nxt_uint_t page_size);
static void nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot,
    nxt_uint_t page_size);
static intptr_t nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1,
    nxt_rbtree_node_t *node2);
static void *nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone,
    nxt_mem_zone_slot_t *slot, size_t size);
static nxt_uint_t nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset,
    nxt_uint_t size);
static void *nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment,
    size_t size);
static nxt_mem_zone_page_t *nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone,
    size_t alignment, uint32_t pages);
static nxt_mem_zone_free_block_t *
    nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node,
    uint32_t alignment, uint32_t pages);
static const char *nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone,
    nxt_mem_zone_page_t *page, void *p);
static void nxt_mem_zone_free_pages(nxt_mem_zone_t *zone,
    nxt_mem_zone_page_t *page, nxt_uint_t count);


static nxt_log_moderation_t  nxt_mem_zone_log_moderation = {
    NXT_LOG_ALERT, 2, "mem_zone_alloc() failed, not enough memory",
    NXT_LOG_MODERATION
};


nxt_mem_zone_t *
nxt_mem_zone_init(u_char *start, size_t zone_size, nxt_uint_t page_size)
{
    uint32_t                   pages;
    nxt_uint_t                 n;
    nxt_mem_zone_t             *zone;
    nxt_mem_zone_page_t        *page;
    nxt_mem_zone_free_block_t  *block;

    if (nxt_slow_path((page_size & (page_size - 1)) != 0)) {
        nxt_thread_log_alert("mem zone page size must be a power of 2");
        return NULL;
    }

    pages = nxt_mem_zone_pages(start, zone_size, page_size);
    if (pages == 0) {
        return NULL;
    }

    zone = (nxt_mem_zone_t *) start;

    /* The function returns address after all slots. */
    page = nxt_mem_zone_slots_init(zone, page_size);

    zone->pages = page;

    for (n = 0; n < pages; n++) {
        page[n].size = NXT_MEM_ZONE_PAGE_FRESH;
    }

    /*
     * A special sentinel page entry marked as used does not correspond
     * to a real page.  The entry simplifies neighbour queue nodes check
     * in nxt_mem_zone_free_pages().
     */
    zone->sentinel_page.size = NXT_MEM_ZONE_PAGE_USED;
    nxt_queue_sentinel(&zone->sentinel_page.link);
    nxt_queue_insert_after(&zone->sentinel_page.link, &page->link);

    /* rbtree of free pages. */

    nxt_rbtree_init(&zone->free_pages, nxt_mem_zone_rbtree_compare);

    block = (nxt_mem_zone_free_block_t *) zone->start;
    block->size = pages;

    nxt_rbtree_insert(&zone->free_pages, &block->node);

    return zone;
}


static uint32_t
nxt_mem_zone_pages(u_char *start, size_t zone_size, nxt_uint_t page_size)
{
    u_char          *end;
    size_t          reserved;
    nxt_uint_t      n, pages, size, chunks, last;
    nxt_mem_zone_t  *zone;

    /*
     * Find all maximum chunk sizes which zone page can be split on
     * with minimum 16-byte step.
     */
    last = page_size / 16;
    n = 0;
    size = 32;

    do {
        chunks = page_size / size;

        if (last != chunks) {
            last = chunks;
            n++;
        }

        size += 16;

    } while (chunks > 1);

    /*
     * Find number of usable zone pages except zone bookkeeping data,
     * slots, and pages entries.
     */
    reserved = sizeof(nxt_mem_zone_t) + (n * sizeof(nxt_mem_zone_slot_t));

    end = nxt_trunc_ptr(start + zone_size, page_size);
    zone_size = end - start;

    pages = (zone_size - reserved) / (page_size + sizeof(nxt_mem_zone_page_t));

    if (reserved > zone_size || pages == 0) {
        nxt_thread_log_alert("mem zone size is too small: %uz", zone_size);
        return 0;
    }

    reserved += pages * sizeof(nxt_mem_zone_page_t);
    nxt_memzero(start, reserved);

    zone = (nxt_mem_zone_t *) start;

    zone->start = nxt_align_ptr(start + reserved, page_size);
    zone->end = end;

    nxt_thread_log_debug("mem zone pages: %uD, unused:%z", pages,
                         end - (zone->start + pages * page_size));

    /*
     * If a chunk size is lesser than zone->small_bitmap_min_size
     * bytes, a page's chunk bitmap is larger than 32 bits and the
     * bimap is placed at the start of the page.
     */
    zone->small_bitmap_min_size = page_size / 32;

    zone->page_size_mask = page_size - 1;
    zone->max_chunk_size = page_size / 2;

    n = zone->max_chunk_size;

    do {
        zone->page_size_shift++;
        n /= 2;
    } while (n != 0);

    return (uint32_t) pages;
}


static void *
nxt_mem_zone_slots_init(nxt_mem_zone_t *zone, nxt_uint_t page_size)
{
    nxt_uint_t           n, size, chunks;
    nxt_mem_zone_slot_t  *slot;

    slot = zone->slots;

    slot[0].chunks = page_size / 16;
    slot[0].size = 16;

    n = 0;
    size = 32;

    for ( ;; ) {
        chunks = page_size / size;

        if (slot[n].chunks != chunks) {

            nxt_mem_zone_slot_init(&slot[n], page_size);

            nxt_thread_log_debug(
                           "mem zone size:%uD chunks:%uD start:%uD map:%uD",
                           slot[n].size, slot[n].chunks + 1,
                           slot[n].start, slot[n].map_size);

            n++;

            if (chunks == 1) {
                return &slot[n];
            }
        }

        slot[n].chunks = chunks;
        slot[n].size = size;
        size += 16;
    }
}


static void
nxt_mem_zone_slot_init(nxt_mem_zone_slot_t *slot, nxt_uint_t page_size)
{
    /*
     * Calculate number of bytes required to store a chunk bitmap
     * and align it to 4 bytes.
     */
    slot->map_size = nxt_align_size(((slot->chunks + 7) / 8), 4);

    /* If chunk size is not a multiple of zone page size, there
     * is surplus space which can be used for the chunk's bitmap.
     */
    slot->start = page_size - slot->chunks * slot->size;

    /* slot->chunks should be one less than actual number of chunks. */
    slot->chunks--;

    if (slot->map_size > 4) {
        /* A page's chunks bitmap is placed at the start of the page. */

        if (slot->start < slot->map_size) {
            /*
             * There is no surplus space or the space is too
             * small for chunks bitmap, so use the first chunks.
             */
            if (slot->size < slot->map_size) {
                /* The first chunks are occupied by bitmap. */
                slot->chunks -= slot->map_size / slot->size;
                slot->start = nxt_align_size(slot->map_size, 16);

            } else {
                /* The first chunk is occupied by bitmap. */
                slot->chunks--;
                slot->start = slot->size;
            }
        }
    }
}


/*
 * Round up to the next highest power of 2.  The algorithm is
 * described in "Bit Twiddling Hacks" by Sean Eron Anderson.
 */

nxt_inline uint32_t
nxt_next_highest_power_of_two(uint32_t n)
{
    n--;
    n |= n >> 1;
    n |= n >> 2;
    n |= n >> 4;
    n |= n >> 8;
    n |= n >> 16;
    n++;

    return n;
}


static intptr_t
nxt_mem_zone_rbtree_compare(nxt_rbtree_node_t *node1, nxt_rbtree_node_t *node2)
{
    u_char                     *start1, *end1, *start2, *end2;
    uint32_t                   n, size, size1, size2;
    nxt_mem_zone_free_block_t  *block1, *block2;

    block1 = (nxt_mem_zone_free_block_t *) node1;
    block2 = (nxt_mem_zone_free_block_t *) node2;

    size1 = block1->size;
    size2 = block2->size;

    /*
     * This subtractions do not overflow if number of pages of a free
     * block is below 2^31-1.  This allows to use blocks up to 128G if
     * a zone page size is just 64 bytes.
     */
    n = size1 - size2;

    if (n != 0) {
        return n;
    }

    /*
     * Sort equally sized blocks by their capability to allocate memory with
     * alignment equal to the size rounded the previous higest power of 2.
     */

    /* Round the size to the previous higest power of two. */
    size = nxt_next_highest_power_of_two(size1) >> 1;

    /* Align the blocks' start and end to the rounded size. */
    start1 = nxt_align_ptr(block1, size);
    end1 = nxt_trunc_ptr((u_char *) block1 + size1, size);

    start2 = nxt_align_ptr(block2, size);
    end2 = nxt_trunc_ptr((u_char *) block2 + size2, size);

    return (end1 - start1) - (end2 - start2);
}


void *
nxt_mem_zone_zalloc(nxt_mem_zone_t *zone, size_t size)
{
    void  *p;

    p = nxt_mem_zone_align(zone, 1, size);

    if (nxt_fast_path(p != NULL)) {
        nxt_memzero(p, size);
    }

    return p;
}


void *
nxt_mem_zone_align(nxt_mem_zone_t *zone, size_t alignment, size_t size)
{
    void                 *p;
    nxt_mem_zone_slot_t  *slot;

    if (nxt_slow_path((alignment - 1) & alignment) != 0) {
        /* Alignment must be a power of 2. */
        return NULL;
    }

    if (size <= zone->max_chunk_size && alignment <= zone->max_chunk_size) {
        /* All chunks are aligned to 16. */

        if (alignment > 16) {
            /*
             * Chunks which size is power of 2 are aligned to the size.
             * So allocation size should be increased to the next highest
             * power of two.  This can waste memory, but a main consumer
             * of aligned allocations is lvlhsh which anyway allocates
             * memory with alignment equal to size.
             */
            size = nxt_next_highest_power_of_two(size);
            size = nxt_max(size, alignment);
        }

        /*
         * Find a zone slot with appropriate chunk size.
         * This operation can be performed without holding lock.
         */
        for (slot = zone->slots; slot->size < size; slot++) { /* void */ }

        nxt_thread_log_debug("mem zone alloc: @%uz:%uz chunk:%uD",
                             alignment, size, slot->size);

        nxt_thread_spin_lock(&zone->lock);

        p = nxt_mem_zone_alloc_small(zone, slot, size);

    } else {

        nxt_thread_log_debug("mem zone alloc: @%uz:%uz", alignment, size);

        nxt_thread_spin_lock(&zone->lock);

        p = nxt_mem_zone_alloc_large(zone, alignment, size);
    }

    nxt_thread_spin_unlock(&zone->lock);

    if (nxt_fast_path(p != NULL)) {
        nxt_thread_log_debug("mem zone alloc: %p", p);

    } else {
        nxt_log_alert_moderate(&nxt_mem_zone_log_moderation, nxt_thread_log(),
                    "nxt_mem_zone_alloc(%uz, %uz) failed, not enough memory",
                    alignment, size);
    }

    return p;
}


static void *
nxt_mem_zone_alloc_small(nxt_mem_zone_t *zone, nxt_mem_zone_slot_t *slot,
    size_t size)
{
    u_char               *p;
    uint8_t              *map;
    nxt_mem_zone_page_t  *page;

    page = slot->pages;

    if (nxt_fast_path(page != NULL)) {

        p = nxt_mem_zone_page_addr(zone, page);

        if (nxt_mem_zone_page_bitmap(zone, slot)) {
            /* A page's chunks bitmap is placed at the start of the page. */
            map = p;

        } else {
            map = page->u.map;
        }

        p += nxt_mem_zone_alloc_chunk(map, slot->start, slot->size);

        page->chunks--;

        if (page->chunks == 0) {
            /*
             * Remove full page from the zone slot list of pages with
             * free chunks.
             */
            slot->pages = page->next;
#if (NXT_DEBUG)
            page->next = NULL;
#endif
        }

        return p;
    }

    page = nxt_mem_zone_alloc_pages(zone, 1, 1);

    if (nxt_fast_path(page != NULL)) {

        slot->pages = page;

        page->size = slot->size;
        /* slot->chunks are already one less. */
        page->chunks = slot->chunks;
        page->u.count = 0;
        page->next = NULL;

        p = nxt_mem_zone_page_addr(zone, page);

        if (nxt_mem_zone_page_bitmap(zone, slot)) {
            /* A page's chunks bitmap is placed at the start of the page. */
            map = p;
            nxt_memzero(map, slot->map_size);

        } else {
            map = page->u.map;
        }

        /* Mark the first chunk as busy. */
        map[0] = 0x80;

        return p + slot->start;
    }

    return NULL;
}


static nxt_uint_t
nxt_mem_zone_alloc_chunk(uint8_t *map, nxt_uint_t offset, nxt_uint_t size)
{
    uint8_t     mask;
    nxt_uint_t  n;

    n = 0;

    /* The page must have at least one free chunk. */

    for ( ;; ) {
        /* The bitmap is always aligned to uint32_t. */

        if (*(uint32_t *) &map[n] != 0xFFFFFFFF) {

            do {
                if (map[n] != 0xFF) {

                    mask = 0x80;

                    do {
                        if ((map[n] & mask) == 0) {
                            /* The free chunk is found. */
                            map[n] |= mask;
                            return offset;
                        }

                        offset += size;
                        mask >>= 1;

                    } while (mask != 0);

                } else {
                    /* Fast-forward: all 8 chunks are occupied. */
                    offset += size * 8;
                }

                n++;

            } while (n % 4 != 0);

        } else {
            /* Fast-forward: all 32 chunks are occupied. */
            offset += size * 32;
            n += 4;
        }
    }
}


static void *
nxt_mem_zone_alloc_large(nxt_mem_zone_t *zone, size_t alignment, size_t size)
{
    uint32_t             pages;
    nxt_mem_zone_page_t  *page;

    pages = (size + zone->page_size_mask) >> zone->page_size_shift;

    page = nxt_mem_zone_alloc_pages(zone, alignment, pages);

    if (nxt_fast_path(page != NULL)) {
        return nxt_mem_zone_page_addr(zone, page);
    }

    return NULL;
}


static nxt_mem_zone_page_t *
nxt_mem_zone_alloc_pages(nxt_mem_zone_t *zone, size_t alignment, uint32_t pages)
{
    u_char                     *p;
    size_t                     prev_size;
    uint32_t                   prev_pages, node_pages, next_pages;
    nxt_uint_t                 n;
    nxt_mem_zone_page_t        *prev_page, *page, *next_page;
    nxt_mem_zone_free_block_t  *block, *next_block;

    block = nxt_mem_zone_find_free_block(zone,
                                         nxt_rbtree_root(&zone->free_pages),
                                         alignment, pages);

    if (nxt_slow_path(block == NULL)) {
        return NULL;
    }

    node_pages = block->size;

    nxt_rbtree_delete(&zone->free_pages, &block->node);

    p = nxt_align_ptr(block, alignment);
    page = nxt_mem_zone_addr_page(zone, p);

    prev_size = p - (u_char *) block;

    if (prev_size != 0) {
        prev_pages = prev_size >>= zone->page_size_shift;
        node_pages -= prev_pages;

        block->size = prev_pages;
        nxt_rbtree_insert(&zone->free_pages, &block->node);

        prev_page = nxt_mem_zone_addr_page(zone, block);
        nxt_queue_insert_after(&prev_page->link, &page->link);
    }

    next_pages = node_pages - pages;

    if (next_pages != 0) {
        next_page = &page[pages];
        next_block = nxt_mem_zone_page_addr(zone, next_page);
        next_block->size = next_pages;

        nxt_rbtree_insert(&zone->free_pages, &next_block->node);
        nxt_queue_insert_after(&page->link, &next_page->link);
    }

    /* Go through pages after all rbtree operations to not trash CPU cache. */

    page[0].u.count = pages;

    for (n = 0; n < pages; n++) {

        if (page[n].size == NXT_MEM_ZONE_PAGE_FRESH) {
            nxt_mem_zone_fresh_junk(nxt_mem_zone_page_addr(zone, &page[n]),
                                    zone->page_size_mask + 1);
        }

        page[n].size = NXT_MEM_ZONE_PAGE_USED;
    }

    return page;
}


/*
 * Free blocks are sorted by size and then if the sizes are equal
 * by aligned allocation capabilty.  The former criterion is just
 * comparison with a requested size and it can be used for iteractive
 * search.  The later criterion cannot be tested only by the requested
 * size and alignment, so recursive in-order tree traversal is required
 * to find a suitable free block.  nxt_mem_zone_find_free_block() uses
 * only recursive in-order tree traversal because anyway the slowest part
 * of the algorithm are CPU cache misses.  Besides the last tail recursive
 * call may be optimized by compiler into iteractive search.
 */

static nxt_mem_zone_free_block_t *
nxt_mem_zone_find_free_block(nxt_mem_zone_t *zone, nxt_rbtree_node_t *node,
    uint32_t alignment, uint32_t pages)
{
    u_char                     *aligned, *end;
    nxt_mem_zone_free_block_t  *block, *free_block;

    if (node == nxt_rbtree_sentinel(&zone->free_pages)) {
        return NULL;
    }

    block = (nxt_mem_zone_free_block_t *) node;

    if (pages <= block->size) {

        free_block = nxt_mem_zone_find_free_block(zone, block->node.left,
                                                  alignment, pages);
        if (free_block != NULL) {
            return free_block;
        }

        aligned = nxt_align_ptr(block, alignment);

        if (pages == block->size) {
            if (aligned == (u_char *) block) {
                /* Exact match. */
                return block;
            }

        } else {  /* pages < block->size */
            aligned += pages << zone->page_size_shift;
            end = nxt_pointer_to(block, block->size << zone->page_size_shift);

            if (aligned <= end) {
                return block;
            }
        }
    }

    return nxt_mem_zone_find_free_block(zone, block->node.right,
                                        alignment, pages);
}


void
nxt_mem_zone_free(nxt_mem_zone_t *zone, void *p)
{
    nxt_uint_t           count;
    const char           *err;
    nxt_mem_zone_page_t  *page;

    nxt_thread_log_debug("mem zone free: %p", p);

    if (nxt_fast_path(zone->start <= (u_char *) p
                      && (u_char *) p < zone->end))
    {
        page = nxt_mem_zone_addr_page(zone, p);

        nxt_thread_spin_lock(&zone->lock);

        if (nxt_mem_zone_page_is_chunked(page)) {
            err = nxt_mem_zone_free_chunk(zone, page, p);

        } else if (nxt_slow_path(nxt_mem_zone_page_is_free(page))) {
            err = "page is already free";

        } else if (nxt_slow_path((uintptr_t) p & zone->page_size_mask) != 0) {
            err = "invalid pointer to chunk";

        } else {
            count = page->u.count;

            if (nxt_fast_path(count != 0)) {
                nxt_mem_zone_free_junk(p, count * zone->page_size_mask + 1);
                nxt_mem_zone_free_pages(zone, page, count);
                err = NULL;

            } else {
                /* Not the first allocated page. */
                err = "pointer to wrong page";
            }
        }

        nxt_thread_spin_unlock(&zone->lock);

    } else {
        err = "pointer is out of zone";
    }

    if (nxt_slow_path(err != NULL)) {
        nxt_thread_log_alert("nxt_mem_zone_free(%p): %s", p, err);
    }
}


static const char *
nxt_mem_zone_free_chunk(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page,
    void *p)
{
    u_char               *map;
    uint32_t             size, offset, chunk;
    nxt_mem_zone_page_t  *pg, **ppg;
    nxt_mem_zone_slot_t  *slot;

    size = page->size;

    /* Find a zone slot with appropriate chunk size. */
    for (slot = zone->slots; slot->size < size; slot++) { /* void */ }

    offset = (uintptr_t) p & zone->page_size_mask;
    offset -= slot->start;

    chunk = offset / size;

    if (nxt_slow_path(offset != chunk * size)) {
        return "pointer to wrong chunk";
    }

    if (nxt_mem_zone_page_bitmap(zone, slot)) {
        /* A page's chunks bitmap is placed at the start of the page. */
        map = (u_char *) ((uintptr_t) p & ~((uintptr_t) zone->page_size_mask));

    } else {
        map = page->u.map;
    }

    if (nxt_mem_zone_chunk_is_free(map, chunk)) {
        return "chunk is already free";
    }

    nxt_mem_zone_set_chunk_free(map, chunk);

    nxt_mem_zone_free_junk(p, page->size);

    if (page->chunks == 0) {
        page->chunks = 1;

        /* Add the page to the head of slot list of pages with free chunks. */
        page->next = slot->pages;
        slot->pages = page;

    } else if (page->chunks != slot->chunks) {
        page->chunks++;

    } else {

        if (map != page->u.map) {
            nxt_mem_zone_free_junk(map, slot->map_size);
        }

        /*
         * All chunks are free, remove the page from the slot list of pages
         * with free chunks and add the page to the free pages tree.
         */
        ppg = &slot->pages;

        for (pg = slot->pages; pg != NULL; pg = pg->next) {

            if (pg == page) {
                *ppg = page->next;
                break;
            }

            ppg = &pg->next;
        }

        nxt_mem_zone_free_pages(zone, page, 1);
    }

    return NULL;
}


static void
nxt_mem_zone_free_pages(nxt_mem_zone_t *zone, nxt_mem_zone_page_t *page,
    nxt_uint_t count)
{
    nxt_mem_zone_page_t        *prev_page, *next_page;
    nxt_mem_zone_free_block_t  *block, *prev_block, *next_block;

    page->size = NXT_MEM_ZONE_PAGE_FREE;
    page->chunks = 0;
    page->u.count = 0;
    page->next = NULL;

    nxt_memzero(&page[1], (count - 1) * sizeof(nxt_mem_zone_page_t));

    next_page = nxt_queue_link_data(page->link.next, nxt_mem_zone_page_t, link);

    if (nxt_mem_zone_page_is_free(next_page)) {

        /* Coalesce with the next free pages. */

        nxt_queue_remove(&next_page->link);
        nxt_memzero(next_page, sizeof(nxt_mem_zone_page_t));

        next_block = nxt_mem_zone_page_addr(zone, next_page);
        count += next_block->size;
        nxt_rbtree_delete(&zone->free_pages, &next_block->node);
    }

    prev_page = nxt_queue_link_data(page->link.prev, nxt_mem_zone_page_t, link);

    if (nxt_mem_zone_page_is_free(prev_page)) {

        /* Coalesce with the previous free pages. */

        nxt_queue_remove(&page->link);

        prev_block = nxt_mem_zone_page_addr(zone, prev_page);
        count += prev_block->size;
        nxt_rbtree_delete(&zone->free_pages, &prev_block->node);

        prev_block->size = count;
        nxt_rbtree_insert(&zone->free_pages, &prev_block->node);

        return;
    }

    block = nxt_mem_zone_page_addr(zone, page);
    block->size = count;
    nxt_rbtree_insert(&zone->free_pages, &block->node);
}