diff --git a/jerry-core/config.h b/jerry-core/config.h index 103672737..c0856ac6e 100644 --- a/jerry-core/config.h +++ b/jerry-core/config.h @@ -26,11 +26,6 @@ */ #define CONFIG_MEM_STACK_LIMIT (4096) -/** - * Log2 of maximum number of chunks in a pool - */ -#define CONFIG_MEM_POOL_MAX_CHUNKS_NUMBER_LOG (8) - /** * Size of pool chunk * @@ -38,11 +33,6 @@ */ #define CONFIG_MEM_POOL_CHUNK_SIZE (8) -/** - * Minimum number of chunks in a pool allocated by pools' manager. - */ -#define CONFIG_MEM_LEAST_CHUNK_NUMBER_IN_POOL (32) - /** * Size of heap chunk */ diff --git a/jerry-core/mem/.mem-poolman.cpp.swn b/jerry-core/mem/.mem-poolman.cpp.swn new file mode 100644 index 000000000..3646f74a7 Binary files /dev/null and b/jerry-core/mem/.mem-poolman.cpp.swn differ diff --git a/jerry-core/mem/mem-allocator.cpp b/jerry-core/mem/mem-allocator.cpp index dc5c7a0f4..08dca6e62 100644 --- a/jerry-core/mem/mem-allocator.cpp +++ b/jerry-core/mem/mem-allocator.cpp @@ -166,6 +166,8 @@ mem_run_try_to_give_memory_back_callbacks (mem_try_give_memory_back_severity_t s { mem_try_give_memory_back_callback (severity); } + + mem_pools_collect_empty (); } /* mem_run_try_to_give_memory_back_callbacks */ #ifndef JERRY_NDEBUG diff --git a/jerry-core/mem/mem-config.h b/jerry-core/mem/mem-config.h index 828252479..781fc08ed 100644 --- a/jerry-core/mem/mem-config.h +++ b/jerry-core/mem/mem-config.h @@ -38,11 +38,6 @@ */ #define MEM_POOL_CHUNK_SIZE ((size_t) (CONFIG_MEM_POOL_CHUNK_SIZE)) -/** - * Log2 of maximum number of chunks in a pool - */ -#define MEM_POOL_MAX_CHUNKS_NUMBER_LOG (CONFIG_MEM_POOL_MAX_CHUNKS_NUMBER_LOG) - /** * Logarithm of required alignment for allocated units/blocks */ diff --git a/jerry-core/mem/mem-pool.cpp b/jerry-core/mem/mem-pool.cpp deleted file mode 100644 index 1acd32bf0..000000000 --- a/jerry-core/mem/mem-pool.cpp +++ /dev/null @@ -1,215 +0,0 @@ -/* Copyright 2014-2015 Samsung Electronics Co., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** \addtogroup mem Memory allocation - * @{ - * - * \addtogroup pool Memory pool - * @{ - */ - -/** - * Memory pool implementation - */ - -#define JERRY_MEM_POOL_INTERNAL - -#include "jrt.h" -#include "jrt-libc-includes.h" -#include "mem-allocator.h" -#include "mem-pool.h" - -/* - * Valgrind-related options and headers - */ -#ifdef JERRY_VALGRIND -# include "memcheck.h" - -# define VALGRIND_NOACCESS_SPACE(p, s) (void)VALGRIND_MAKE_MEM_NOACCESS((p), (s)) -# define VALGRIND_UNDEFINED_SPACE(p, s) (void)VALGRIND_MAKE_MEM_UNDEFINED((p), (s)) -# define VALGRIND_DEFINED_SPACE(p, s) (void)VALGRIND_MAKE_MEM_DEFINED((p), (s)) -#else /* JERRY_VALGRIND */ -# define VALGRIND_NOACCESS_SPACE(p, s) -# define VALGRIND_UNDEFINED_SPACE(p, s) -# define VALGRIND_DEFINED_SPACE(p, s) -#endif /* JERRY_VALGRIND */ - -static void mem_check_pool (mem_pool_state_t *pool_p); - -/** - * Get address of pool chunk with specified index - */ -#define MEM_POOL_CHUNK_ADDRESS(pool_header_p, chunk_index) ((uint8_t*) (MEM_POOL_SPACE_START(pool_p) + \ - MEM_POOL_CHUNK_SIZE * chunk_index)) - -/** - * Is the chunk is inside of the pool? - * - * @return true / false - */ -bool __attr_const___ -mem_pool_is_chunk_inside (mem_pool_state_t *pool_p, /**< pool */ - uint8_t *chunk_p) /**< chunk */ -{ - if (chunk_p >= (uint8_t*) pool_p && chunk_p < (uint8_t*) pool_p + MEM_POOL_SIZE) - { - JERRY_ASSERT (chunk_p >= MEM_POOL_SPACE_START (pool_p) - && chunk_p <= MEM_POOL_SPACE_START (pool_p) + MEM_POOL_CHUNKS_NUMBER * MEM_POOL_CHUNK_SIZE); - - return true; - } - - return false; -} /* mem_pool_is_chunk_inside */ - -/** - * Initialization of memory pool. - * - * Pool will be located in the segment [pool_start; pool_start + pool_size). - * Part of pool space will be used for bitmap and the rest will store chunks. - */ -void -mem_pool_init (mem_pool_state_t *pool_p, /**< pool */ - size_t pool_size) /**< pool size */ -{ - JERRY_ASSERT (pool_p != NULL); - JERRY_ASSERT ((size_t)MEM_POOL_SPACE_START (pool_p) % MEM_ALIGNMENT == 0); - - JERRY_STATIC_ASSERT (MEM_POOL_CHUNK_SIZE % MEM_ALIGNMENT == 0); - JERRY_STATIC_ASSERT (MEM_POOL_MAX_CHUNKS_NUMBER_LOG <= sizeof (mem_pool_chunk_index_t) * JERRY_BITSINBYTE); - JERRY_ASSERT (sizeof (mem_pool_chunk_index_t) <= MEM_POOL_CHUNK_SIZE); - - JERRY_ASSERT (MEM_POOL_SIZE == sizeof (mem_pool_state_t) + MEM_POOL_CHUNKS_NUMBER * MEM_POOL_CHUNK_SIZE); - JERRY_ASSERT (MEM_POOL_CHUNKS_NUMBER >= CONFIG_MEM_LEAST_CHUNK_NUMBER_IN_POOL); - - JERRY_ASSERT (pool_size == MEM_POOL_SIZE); - - /* - * All chunks are free right after initialization - */ - pool_p->free_chunks_number = (mem_pool_chunk_index_t) MEM_POOL_CHUNKS_NUMBER; - JERRY_ASSERT (pool_p->free_chunks_number == MEM_POOL_CHUNKS_NUMBER); - - /* - * Chunk with zero index is first free chunk in the pool now - */ - pool_p->first_free_chunk = 0; - - for (mem_pool_chunk_index_t chunk_index = 0; - chunk_index < MEM_POOL_CHUNKS_NUMBER; - chunk_index++) - { - mem_pool_chunk_index_t *next_free_chunk_index_p = (mem_pool_chunk_index_t*) MEM_POOL_CHUNK_ADDRESS (pool_p, - chunk_index); - - *next_free_chunk_index_p = (mem_pool_chunk_index_t) (chunk_index + 1u); - - VALGRIND_NOACCESS_SPACE (next_free_chunk_index_p, MEM_POOL_CHUNK_SIZE); - } - - mem_check_pool (pool_p); -} /* mem_pool_init */ - -/** - * Allocate a chunk in the pool - */ -uint8_t* -mem_pool_alloc_chunk (mem_pool_state_t *pool_p) /**< pool */ -{ - mem_check_pool (pool_p); - - JERRY_ASSERT (pool_p->free_chunks_number != 0); - JERRY_ASSERT (pool_p->first_free_chunk < MEM_POOL_CHUNKS_NUMBER); - - mem_pool_chunk_index_t chunk_index = pool_p->first_free_chunk; - uint8_t *chunk_p = MEM_POOL_CHUNK_ADDRESS (pool_p, chunk_index); - - VALGRIND_DEFINED_SPACE (chunk_p, MEM_POOL_CHUNK_SIZE); - - mem_pool_chunk_index_t *next_free_chunk_index_p = (mem_pool_chunk_index_t*) chunk_p; - pool_p->first_free_chunk = *next_free_chunk_index_p; - pool_p->free_chunks_number--; - - VALGRIND_UNDEFINED_SPACE (chunk_p, MEM_POOL_CHUNK_SIZE); - - mem_check_pool (pool_p); - - return chunk_p; -} /* mem_pool_alloc_chunk */ - -/** - * Free the chunk in the pool - */ -void -mem_pool_free_chunk (mem_pool_state_t *pool_p, /**< pool */ - uint8_t *chunk_p) /**< chunk pointer */ -{ - JERRY_ASSERT (pool_p->free_chunks_number < MEM_POOL_CHUNKS_NUMBER); - JERRY_ASSERT (mem_pool_is_chunk_inside (pool_p, chunk_p)); - JERRY_ASSERT (((uintptr_t) chunk_p - (uintptr_t) MEM_POOL_SPACE_START (pool_p)) % MEM_POOL_CHUNK_SIZE == 0); - - mem_check_pool (pool_p); - - const size_t chunk_byte_offset = (size_t) (chunk_p - MEM_POOL_SPACE_START (pool_p)); - const mem_pool_chunk_index_t chunk_index = (mem_pool_chunk_index_t) (chunk_byte_offset / MEM_POOL_CHUNK_SIZE); - - mem_pool_chunk_index_t *next_free_chunk_index_p = (mem_pool_chunk_index_t*) chunk_p; - - *next_free_chunk_index_p = pool_p->first_free_chunk; - - pool_p->first_free_chunk = chunk_index; - pool_p->free_chunks_number++; - - VALGRIND_NOACCESS_SPACE (next_free_chunk_index_p, MEM_POOL_CHUNK_SIZE); - - mem_check_pool (pool_p); -} /* mem_pool_free_chunk */ - -/** - * Check pool state consistency - */ -static void -mem_check_pool (mem_pool_state_t __attr_unused___ *pool_p) /**< pool (unused #ifdef JERRY_DISABLE_HEAVY_DEBUG) */ -{ -#ifndef JERRY_DISABLE_HEAVY_DEBUG - JERRY_ASSERT (pool_p->free_chunks_number <= MEM_POOL_CHUNKS_NUMBER); - - size_t met_free_chunks_number = 0; - mem_pool_chunk_index_t chunk_index = pool_p->first_free_chunk; - - while (chunk_index != MEM_POOL_CHUNKS_NUMBER) - { - uint8_t *chunk_p = MEM_POOL_CHUNK_ADDRESS (pool_p, chunk_index); - mem_pool_chunk_index_t *next_free_chunk_index_p = (mem_pool_chunk_index_t*) chunk_p; - - met_free_chunks_number++; - - VALGRIND_DEFINED_SPACE (next_free_chunk_index_p, MEM_POOL_CHUNK_SIZE); - - chunk_index = *next_free_chunk_index_p; - - VALGRIND_NOACCESS_SPACE (next_free_chunk_index_p, MEM_POOL_CHUNK_SIZE); - } - - JERRY_ASSERT (met_free_chunks_number == pool_p->free_chunks_number); -#else /* !JERRY_DISABLE_HEAVY_DEBUG */ - (void) pool_p; -#endif /* JERRY_DISABLE_HEAVY_DEBUG */ -} /* mem_check_pool */ - -/** - * @} - * @} - */ diff --git a/jerry-core/mem/mem-pool.h b/jerry-core/mem/mem-pool.h deleted file mode 100644 index 731eb111b..000000000 --- a/jerry-core/mem/mem-pool.h +++ /dev/null @@ -1,80 +0,0 @@ -/* Copyright 2014-2015 Samsung Electronics Co., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef JERRY_MEM_POOL_INTERNAL -#error "Please, use mem_poolman.h instead of mem_pool.h" -#endif - -#ifndef JERRY_MEM_POOL_H -#define JERRY_MEM_POOL_H - -#include "mem-config.h" -#include "mem-heap.h" - -/** \addtogroup pool Memory pool - * @{ - */ - -/** - * Size of a pool (header + chunks) - */ -#define MEM_POOL_SIZE \ - ((size_t) JERRY_MIN (sizeof (mem_pool_state_t) + (1ull << MEM_POOL_MAX_CHUNKS_NUMBER_LOG) * MEM_POOL_CHUNK_SIZE, \ - JERRY_ALIGNDOWN (mem_heap_recommend_allocation_size (sizeof (mem_pool_state_t) + \ - CONFIG_MEM_LEAST_CHUNK_NUMBER_IN_POOL * \ - MEM_POOL_CHUNK_SIZE) \ - - sizeof (mem_pool_state_t), \ - MEM_POOL_CHUNK_SIZE)) + sizeof (mem_pool_state_t)) - -/** - * Number of chunks in a pool - */ -#define MEM_POOL_CHUNKS_NUMBER ((MEM_POOL_SIZE - sizeof (mem_pool_state_t)) / MEM_POOL_CHUNK_SIZE) - -/** - * Get pool's space size - */ -#define MEM_POOL_SPACE_START(pool_header_p) ((uint8_t*) ((mem_pool_state_t*) pool_header_p + 1)) - -/** - * Index of chunk in a pool - */ -typedef uint8_t mem_pool_chunk_index_t; - -/** - * State of a memory pool - */ -typedef struct __attribute__ ((aligned (MEM_ALIGNMENT))) mem_pool_state_t -{ - /** Offset of first free chunk from the beginning of the pool (mem_pool_chunk_index_t) */ - mem_pool_chunk_index_t first_free_chunk : MEM_POOL_MAX_CHUNKS_NUMBER_LOG; - - /** Number of free chunks (mem_pool_chunk_index_t) */ - mem_pool_chunk_index_t free_chunks_number : MEM_POOL_MAX_CHUNKS_NUMBER_LOG; - - /** Pointer to the next pool with same chunk size */ - mem_cpointer_t next_pool_cp : MEM_CP_WIDTH; -} mem_pool_state_t; - -extern void mem_pool_init (mem_pool_state_t *pool_p, size_t pool_size); -extern uint8_t* mem_pool_alloc_chunk (mem_pool_state_t *pool_p); -extern void mem_pool_free_chunk (mem_pool_state_t *pool_p, uint8_t *chunk_p); -extern bool __attr_const___ mem_pool_is_chunk_inside (mem_pool_state_t *pool_p, uint8_t *chunk_p); - -/** - * @} - */ - -#endif /* JERRY_MEM_POOL_H */ diff --git a/jerry-core/mem/mem-poolman.cpp b/jerry-core/mem/mem-poolman.cpp index 24936fb9e..70df2be44 100644 --- a/jerry-core/mem/mem-poolman.cpp +++ b/jerry-core/mem/mem-poolman.cpp @@ -30,18 +30,79 @@ #include "jrt-libc-includes.h" #include "mem-allocator.h" #include "mem-heap.h" -#include "mem-pool.h" #include "mem-poolman.h" /** - * Lists of pools + * Size of a pool */ -mem_pool_state_t *mem_pools; +#define MEM_POOL_SIZE (mem_heap_get_chunked_block_data_size ()) /** - * Number of free chunks + * Number of chunks in a pool */ +#define MEM_POOL_CHUNKS_NUMBER (MEM_POOL_SIZE / MEM_POOL_CHUNK_SIZE) + +#ifndef JERRY_NDEBUG size_t mem_free_chunks_number; +#endif /* !JERRY_NDEBUG */ + +/** + * Index of chunk in a pool + */ +typedef uint8_t mem_pool_chunk_index_t; + +/** + * Pool chunk + */ +typedef struct mem_pool_chunk_t +{ + /** + * Union of possible free chunk layouts + * + * Allocated chunk represents raw data of MEM_POOL_CHUNK_SIZE bytes, + * and so, has no fixed layout. + */ + union + { + /** + * Structure of free pool chunks that are: + * - first in corresponding pool, while empty pool collector is not active; + * - not first in corresponding pool. + */ + struct + { + mem_pool_chunk_t *next_p; /**< global list of free pool chunks */ + } free; + + /** + * While empty pool collector is active, the following structure is used + * for first chunks of pools, in which first chunks are free + * + * See also: + * mem_pools_collect_empty + */ + struct + { + mem_cpointer_t next_first_cp; /**< list of first free chunks of + * pools with free first chunks */ + mem_cpointer_t free_list_cp; /**< list of free chunks + * in the pool containing this chunk */ + uint16_t hint_magic_num; /**< magic number that hints whether + * there is a probability that the chunk + * is an item (header) in a pool list */ + mem_pool_chunk_index_t free_chunks_num; /**< number of free chunks + * in the pool containing this chunk */ + uint8_t list_id; /**< identifier of a pool list */ + } pool_gc; + } u; +} mem_pool_chunk_t; + +/** + * List of free pool chunks + */ +mem_pool_chunk_t *mem_free_chunk_p; + +static void mem_check_pools (void); #ifdef MEM_STATS /** @@ -68,14 +129,32 @@ static void mem_pools_stat_free_chunk (void); # define MEM_POOLS_STAT_FREE_CHUNK() #endif /* !MEM_STATS */ +/* + * Valgrind-related options and headers + */ +#ifdef JERRY_VALGRIND +# include "memcheck.h" + +# define VALGRIND_NOACCESS_SPACE(p, s) (void)VALGRIND_MAKE_MEM_NOACCESS((p), (s)) +# define VALGRIND_UNDEFINED_SPACE(p, s) (void)VALGRIND_MAKE_MEM_UNDEFINED((p), (s)) +# define VALGRIND_DEFINED_SPACE(p, s) (void)VALGRIND_MAKE_MEM_DEFINED((p), (s)) +#else /* JERRY_VALGRIND */ +# define VALGRIND_NOACCESS_SPACE(p, s) +# define VALGRIND_UNDEFINED_SPACE(p, s) +# define VALGRIND_DEFINED_SPACE(p, s) +#endif /* JERRY_VALGRIND */ + /** * Initialize pool manager */ void mem_pools_init (void) { - mem_pools = NULL; +#ifndef JERRY_NDEBUG mem_free_chunks_number = 0; +#endif /* !JERRY_NDEBUG */ + + mem_free_chunk_p = NULL; MEM_POOLS_STAT_INIT (); } /* mem_pools_init */ @@ -86,61 +165,327 @@ mem_pools_init (void) void mem_pools_finalize (void) { - JERRY_ASSERT (mem_pools == NULL); + mem_pools_collect_empty (); + +#ifndef JERRY_NDEBUG JERRY_ASSERT (mem_free_chunks_number == 0); +#endif /* !JERRY_NDEBUG */ } /* mem_pools_finalize */ +void +mem_pools_collect_empty (void) +{ + /* + * Hint magic number in header of pools with free first chunks + */ + const uint16_t hint_magic_num_value = 0x7e89; + + /* + * At first pass collect pointers to those of free chunks that are first at their pools + * to separate lists (collection-time pool lists) and change them to headers of corresponding pools + */ + + /* + * Number of collection-time pool lists + */ + constexpr uint32_t pool_lists_number = 8; + + /* + * Collection-time pool lists + */ + mem_pool_chunk_t *pool_lists_p[pool_lists_number]; + for (uint32_t i = 0; i < pool_lists_number; i++) + { + pool_lists_p[i] = NULL; + } + + /* + * Number of the pools, included into the lists + */ + uint32_t pools_in_lists_number = 0; + + for (mem_pool_chunk_t *free_chunk_iter_p = mem_free_chunk_p, *prev_free_chunk_p = NULL, *next_free_chunk_p; + free_chunk_iter_p != NULL; + free_chunk_iter_p = next_free_chunk_p) + { + mem_pool_chunk_t *pool_start_p = (mem_pool_chunk_t *) mem_heap_get_chunked_block_start (free_chunk_iter_p); + + VALGRIND_DEFINED_SPACE (free_chunk_iter_p, MEM_POOL_CHUNK_SIZE); + + next_free_chunk_p = free_chunk_iter_p->u.free.next_p; + + if (pool_start_p == free_chunk_iter_p) + { + /* + * The chunk is first at its pool + * + * Remove the chunk from common list of free chunks + */ + if (prev_free_chunk_p == NULL) + { + JERRY_ASSERT (mem_free_chunk_p == free_chunk_iter_p); + + mem_free_chunk_p = next_free_chunk_p; + } + else + { + prev_free_chunk_p->u.free.next_p = next_free_chunk_p; + } + + pools_in_lists_number++; + + uint8_t list_id = pools_in_lists_number % pool_lists_number; + + /* + * Initialize pool header and insert the pool into one of lists + */ + free_chunk_iter_p->u.pool_gc.free_list_cp = MEM_CP_NULL; + free_chunk_iter_p->u.pool_gc.free_chunks_num = 1; /* the first chunk */ + free_chunk_iter_p->u.pool_gc.hint_magic_num = hint_magic_num_value; + free_chunk_iter_p->u.pool_gc.list_id = list_id; + + MEM_CP_SET_POINTER (free_chunk_iter_p->u.pool_gc.next_first_cp, pool_lists_p[list_id]); + pool_lists_p[list_id] = free_chunk_iter_p; + } + else + { + prev_free_chunk_p = free_chunk_iter_p; + } + } + + if (pools_in_lists_number == 0) + { + /* there are no empty pools */ + + return; + } + + /* + * At second pass we check for all rest free chunks whether they are in pools that were included into + * collection-time pool lists. + * + * For each of the chunk, try to find the corresponding pool through iterating the list. + * + * If pool is found in a list (so, first chunk of the pool is free) for a chunk, increment counter + * of free chunks in the pools, and move the chunk from global free chunks list to collection-time + * local list of corresponding pool's free chunks. + */ + for (mem_pool_chunk_t *free_chunk_iter_p = mem_free_chunk_p, *prev_free_chunk_p = NULL, *next_free_chunk_p; + free_chunk_iter_p != NULL; + free_chunk_iter_p = next_free_chunk_p) + { + mem_pool_chunk_t *pool_start_p = (mem_pool_chunk_t *) mem_heap_get_chunked_block_start (free_chunk_iter_p); + + next_free_chunk_p = free_chunk_iter_p->u.free.next_p; + + bool is_chunk_moved_to_local_list = false; + + /* + * The magic number doesn't guarantee that the chunk is actually a pool header, + * so it is only optimization to reduce number of unnecessary iterations over + * pool lists. + */ + if (pool_start_p->u.pool_gc.hint_magic_num == hint_magic_num_value) + { + /* + * Maybe, the first chunk is free. + * + * If it is so, it is included in the list of pool's first free chunks. + */ + uint8_t id_to_search_in = pool_start_p->u.pool_gc.list_id; + + if (id_to_search_in < pool_lists_number) + { + for (mem_pool_chunk_t *pool_list_iter_p = pool_lists_p[id_to_search_in]; + pool_list_iter_p != NULL; + pool_list_iter_p = MEM_CP_GET_POINTER (mem_pool_chunk_t, + pool_list_iter_p->u.pool_gc.next_first_cp)) + { + if (pool_list_iter_p == pool_start_p) + { + /* + * The first chunk is actually free. + * + * So, incrementing free chunks counter in it. + */ + pool_start_p->u.pool_gc.free_chunks_num++; + + /* + * It is possible that the corresponding pool is empty + * + * Moving current chunk from common list of free chunks to temporary list, local to the pool + */ + if (prev_free_chunk_p == NULL) + { + JERRY_ASSERT (mem_free_chunk_p == free_chunk_iter_p); + + mem_free_chunk_p = next_free_chunk_p; + } + else + { + prev_free_chunk_p->u.free.next_p = next_free_chunk_p; + } + + free_chunk_iter_p->u.free.next_p = MEM_CP_GET_POINTER (mem_pool_chunk_t, + pool_start_p->u.pool_gc.free_list_cp); + MEM_CP_SET_NON_NULL_POINTER (pool_start_p->u.pool_gc.free_list_cp, free_chunk_iter_p); + + is_chunk_moved_to_local_list = true; + + break; + } + } + } + } + + if (!is_chunk_moved_to_local_list) + { + prev_free_chunk_p = free_chunk_iter_p; + } + } + + /* + * At third pass we check each pool in collection-time pool lists free for counted + * number of free chunks in the pool. + * + * If the number is equal to number of chunks in the pool - then the pool is empty, and so is freed, + * otherwise - free chunks of the pool are returned to common list of free chunks. + */ + for (uint8_t list_id = 0; list_id < pool_lists_number; list_id++) + { + for (mem_pool_chunk_t *pool_list_iter_p = pool_lists_p[list_id], *next_p; + pool_list_iter_p != NULL; + pool_list_iter_p = next_p) + { + next_p = MEM_CP_GET_POINTER (mem_pool_chunk_t, + pool_list_iter_p->u.pool_gc.next_first_cp); + + if (pool_list_iter_p->u.pool_gc.free_chunks_num == MEM_POOL_CHUNKS_NUMBER) + { +#ifndef JERRY_NDEBUG + mem_free_chunks_number -= MEM_POOL_CHUNKS_NUMBER; +#endif /* !JERRY_NDEBUG */ + + mem_heap_free_block (pool_list_iter_p); + + MEM_POOLS_STAT_FREE_POOL (); + } + else + { + mem_pool_chunk_t *first_chunk_p = pool_list_iter_p; + + /* + * Convert layout of first chunk from collection-time pool header to common free chunk + */ + first_chunk_p->u.free.next_p = MEM_CP_GET_POINTER (mem_pool_chunk_t, + pool_list_iter_p->u.pool_gc.free_list_cp); + + /* + * Link local pool's list of free chunks into global list of free chunks + */ + for (mem_pool_chunk_t *pool_chunks_iter_p = first_chunk_p; + ; + pool_chunks_iter_p = pool_chunks_iter_p->u.free.next_p) + { + JERRY_ASSERT (pool_chunks_iter_p != NULL); + + if (pool_chunks_iter_p->u.free.next_p == NULL) + { + pool_chunks_iter_p->u.free.next_p = mem_free_chunk_p; + + break; + } + } + + mem_free_chunk_p = first_chunk_p; + } + } + } + +#ifdef JERRY_VALGRIND + /* + * Valgrind-mode specific pass that marks all free chunks inaccessible + */ + for (mem_pool_chunk_t *free_chunk_iter_p = mem_free_chunk_p, *next_free_chunk_p; + free_chunk_iter_p != NULL; + free_chunk_iter_p = next_free_chunk_p) + { + next_free_chunk_p = free_chunk_iter_p->u.free.next_p; + + VALGRIND_NOACCESS_SPACE (free_chunk_iter_p, MEM_POOL_CHUNK_SIZE); + } +#endif /* JERRY_VALGRIND */ +} /* mem_pools_collect_empty */ + /** * Long path for mem_pools_alloc - * - * @return true - if there is a free chunk in mem_pools, - * false - otherwise (not enough memory). */ -static bool __attr_noinline___ +static void __attr_noinline___ mem_pools_alloc_longpath (void) { - /** - * If there are no free chunks, allocate new pool. - */ - if (mem_free_chunks_number == 0) + mem_check_pools (); + + JERRY_ASSERT (mem_free_chunk_p == NULL); + + JERRY_ASSERT (MEM_POOL_SIZE <= mem_heap_get_chunked_block_data_size ()); + JERRY_ASSERT (MEM_POOL_CHUNKS_NUMBER >= 1); + + mem_pool_chunk_t *pool_start_p = (mem_pool_chunk_t*) mem_heap_alloc_chunked_block (MEM_HEAP_ALLOC_LONG_TERM); + + if (mem_free_chunk_p != NULL) { - mem_pool_state_t *pool_state = (mem_pool_state_t*) mem_heap_alloc_block (MEM_POOL_SIZE, MEM_HEAP_ALLOC_LONG_TERM); + /* some chunks were freed due to GC invoked by heap allocator */ + mem_heap_free_block (pool_start_p); - JERRY_ASSERT (pool_state != NULL); - - mem_pool_init (pool_state, MEM_POOL_SIZE); - - MEM_CP_SET_POINTER (pool_state->next_pool_cp, mem_pools); - - mem_pools = pool_state; - - mem_free_chunks_number += MEM_POOL_CHUNKS_NUMBER; - - MEM_POOLS_STAT_ALLOC_POOL (); + return; } - else - { - /** - * There is definitely at least one pool of specified type with at least one free chunk. - * - * Search for the pool. - */ - mem_pool_state_t *pool_state = mem_pools, *prev_pool_state_p = NULL; - while (pool_state->first_free_chunk == MEM_POOL_CHUNKS_NUMBER) +#ifndef JERRY_NDEBUG + mem_free_chunks_number += MEM_POOL_CHUNKS_NUMBER; +#endif /* !JERRY_NDEBUG */ + + JERRY_STATIC_ASSERT (MEM_POOL_CHUNK_SIZE % MEM_ALIGNMENT == 0); + JERRY_STATIC_ASSERT (sizeof (mem_pool_chunk_t) == MEM_POOL_CHUNK_SIZE); + JERRY_STATIC_ASSERT (sizeof (mem_pool_chunk_index_t) <= MEM_POOL_CHUNK_SIZE); + JERRY_ASSERT ((mem_pool_chunk_index_t) MEM_POOL_CHUNKS_NUMBER == MEM_POOL_CHUNKS_NUMBER); + JERRY_ASSERT (MEM_POOL_SIZE == MEM_POOL_CHUNKS_NUMBER * MEM_POOL_CHUNK_SIZE); + + JERRY_ASSERT (((uintptr_t) pool_start_p) % MEM_ALIGNMENT == 0); + + mem_pool_chunk_t *prev_free_chunk_p = NULL; + + for (mem_pool_chunk_index_t chunk_index = 0; + chunk_index < MEM_POOL_CHUNKS_NUMBER; + chunk_index++) + { + mem_pool_chunk_t *chunk_p = pool_start_p + chunk_index; + + if (prev_free_chunk_p != NULL) { - prev_pool_state_p = pool_state; - pool_state = MEM_CP_GET_NON_NULL_POINTER (mem_pool_state_t, pool_state->next_pool_cp); + prev_free_chunk_p->u.free.next_p = chunk_p; } - JERRY_ASSERT (prev_pool_state_p != NULL && pool_state != mem_pools); - - prev_pool_state_p->next_pool_cp = pool_state->next_pool_cp; - MEM_CP_SET_NON_NULL_POINTER (pool_state->next_pool_cp, mem_pools); - mem_pools = pool_state; + prev_free_chunk_p = chunk_p; } - return true; + prev_free_chunk_p->u.free.next_p = NULL; + +#ifdef JERRY_VALGRIND + for (mem_pool_chunk_index_t chunk_index = 0; + chunk_index < MEM_POOL_CHUNKS_NUMBER; + chunk_index++) + { + mem_pool_chunk_t *chunk_p = pool_start_p + chunk_index; + + VALGRIND_NOACCESS_SPACE (chunk_p, MEM_POOL_CHUNK_SIZE); + } +#endif /* JERRY_VALGRIND */ + + mem_free_chunk_p = pool_start_p; + + MEM_POOLS_STAT_ALLOC_POOL (); + + mem_check_pools (); } /* mem_pools_alloc_longpath */ /** @@ -149,84 +494,94 @@ mem_pools_alloc_longpath (void) * @return pointer to allocated chunk, if allocation was successful, * or NULL - if not enough memory. */ -uint8_t* +uint8_t* __attr_always_inline___ mem_pools_alloc (void) { - if (mem_pools == NULL || mem_pools->first_free_chunk == MEM_POOL_CHUNKS_NUMBER) + mem_check_pools (); + + do { - if (!mem_pools_alloc_longpath ()) + if (mem_free_chunk_p != NULL) { - return NULL; + mem_pool_chunk_t *chunk_p = mem_free_chunk_p; + + MEM_POOLS_STAT_ALLOC_CHUNK (); + +#ifndef JERRY_NDEBUG + mem_free_chunks_number--; +#endif /* !JERRY_NDEBUG */ + + VALGRIND_DEFINED_SPACE (chunk_p, MEM_POOL_CHUNK_SIZE); + + mem_free_chunk_p = chunk_p->u.free.next_p; + + VALGRIND_UNDEFINED_SPACE (chunk_p, MEM_POOL_CHUNK_SIZE); + + + mem_check_pools (); + + return (uint8_t *) chunk_p; } - } + else + { + mem_pools_alloc_longpath (); - JERRY_ASSERT (mem_pools != NULL && mem_pools->first_free_chunk != MEM_POOL_CHUNKS_NUMBER); - - /** - * And allocate chunk within it. - */ - mem_free_chunks_number--; - - MEM_POOLS_STAT_ALLOC_CHUNK (); - - return mem_pool_alloc_chunk (mem_pools); + /* the assertion guarantees that there will be no more than two iterations */ + JERRY_ASSERT (mem_free_chunk_p != NULL); + } + } while (true); } /* mem_pools_alloc */ /** * Free the chunk */ -void +void __attr_always_inline___ mem_pools_free (uint8_t *chunk_p) /**< pointer to the chunk */ { - mem_pool_state_t *pool_state = mem_pools, *prev_pool_state_p = NULL; + mem_check_pools (); - /** - * Search for the pool containing specified chunk. - */ - while (!mem_pool_is_chunk_inside (pool_state, chunk_p)) - { - prev_pool_state_p = pool_state; - pool_state = MEM_CP_GET_NON_NULL_POINTER (mem_pool_state_t, pool_state->next_pool_cp); - } + mem_pool_chunk_t *chunk_to_free_p = (mem_pool_chunk_t *) chunk_p; - /** - * Free the chunk - */ - mem_pool_free_chunk (pool_state, chunk_p); + chunk_to_free_p->u.free.next_p = mem_free_chunk_p; + mem_free_chunk_p = chunk_to_free_p; + + VALGRIND_NOACCESS_SPACE (chunk_to_free_p, MEM_POOL_CHUNK_SIZE); + +#ifndef JERRY_NDEBUG mem_free_chunks_number++; +#endif /* !JERRY_NDEBUG */ MEM_POOLS_STAT_FREE_CHUNK (); - /** - * If all chunks of the pool are free, free the pool itself. - */ - if (pool_state->free_chunks_number == MEM_POOL_CHUNKS_NUMBER) - { - if (prev_pool_state_p != NULL) - { - prev_pool_state_p->next_pool_cp = pool_state->next_pool_cp; - } - else - { - mem_pools = MEM_CP_GET_POINTER (mem_pool_state_t, pool_state->next_pool_cp); - } - - mem_free_chunks_number -= MEM_POOL_CHUNKS_NUMBER; - - mem_heap_free_block ((uint8_t*) pool_state); - - MEM_POOLS_STAT_FREE_POOL (); - } - else if (mem_pools != pool_state) - { - JERRY_ASSERT (prev_pool_state_p != NULL); - - prev_pool_state_p->next_pool_cp = pool_state->next_pool_cp; - MEM_CP_SET_NON_NULL_POINTER (pool_state->next_pool_cp, mem_pools); - mem_pools = pool_state; - } + mem_check_pools (); } /* mem_pools_free */ +/** + * Check correctness of pool allocator state + */ +static void +mem_check_pools (void) +{ +#ifndef JERRY_DISABLE_HEAVY_DEBUG + size_t free_chunks_met = 0; + + for (mem_pool_chunk_t *free_chunk_iter_p = mem_free_chunk_p, *next_free_chunk_p; + free_chunk_iter_p != NULL; + free_chunk_iter_p = next_free_chunk_p) + { + VALGRIND_DEFINED_SPACE (free_chunk_iter_p, MEM_POOL_CHUNK_SIZE); + + next_free_chunk_p = free_chunk_iter_p->u.free.next_p; + + VALGRIND_NOACCESS_SPACE (free_chunk_iter_p, MEM_POOL_CHUNK_SIZE); + + free_chunks_met++; + } + + JERRY_ASSERT (free_chunks_met == mem_free_chunks_number); +#endif /* !JERRY_DISABLE_HEAVY_DEBUG */ +} /* mem_check_pools */ + #ifdef MEM_STATS /** * Get pools memory usage statistics @@ -265,7 +620,6 @@ static void mem_pools_stat_alloc_pool (void) { mem_pools_stats.pools_count++; - mem_pools_stats.free_chunks = mem_free_chunks_number; if (mem_pools_stats.pools_count > mem_pools_stats.peak_pools_count) { @@ -286,7 +640,6 @@ mem_pools_stat_free_pool (void) JERRY_ASSERT (mem_pools_stats.pools_count > 0); mem_pools_stats.pools_count--; - mem_pools_stats.free_chunks = mem_free_chunks_number; } /* mem_pools_stat_free_pool */ /** diff --git a/jerry-core/mem/mem-poolman.h b/jerry-core/mem/mem-poolman.h index 1df452b65..5e175db8c 100644 --- a/jerry-core/mem/mem-poolman.h +++ b/jerry-core/mem/mem-poolman.h @@ -33,6 +33,7 @@ extern void mem_pools_init (void); extern void mem_pools_finalize (void); extern uint8_t* mem_pools_alloc (void); extern void mem_pools_free (uint8_t *chunk_p); +extern void mem_pools_collect_empty (void); #ifdef MEM_STATS /** diff --git a/tests/unit/test-pool.cpp b/tests/unit/test-pool.cpp deleted file mode 100644 index 1c8b8d222..000000000 --- a/tests/unit/test-pool.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2014-2015 Samsung Electronics Co., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define JERRY_MEM_POOL_INTERNAL - -#include "mem-allocator.h" -#include "mem-pool.h" -#include "mem-poolman.h" - -#include "test-common.h" - -// Iterations count -const uint32_t test_iters = 64; - -// Subiterations count -const uint32_t test_max_sub_iters = 1024; - -#define TEST_POOL_SPACE_SIZE (sizeof (mem_pool_state_t) + \ - (1ull << MEM_POOL_MAX_CHUNKS_NUMBER_LOG) * MEM_POOL_CHUNK_SIZE) -uint8_t test_pool[TEST_POOL_SPACE_SIZE] __attribute__ ((aligned (MEM_ALIGNMENT))); - -uint8_t* ptrs[test_max_sub_iters]; - -int -main (int __attr_unused___ argc, - char __attr_unused___ **argv) -{ - TEST_INIT (); - - for (uint32_t i = 0; i < test_iters; i++) - { - mem_pool_state_t* pool_p = (mem_pool_state_t*) test_pool; - - JERRY_ASSERT (MEM_POOL_SIZE <= TEST_POOL_SPACE_SIZE); - - mem_pool_init (pool_p, MEM_POOL_SIZE); - - const size_t subiters = ((size_t) rand () % test_max_sub_iters) + 1; - - for (size_t j = 0; j < subiters; j++) - { - if (pool_p->free_chunks_number != 0) - { - ptrs[j] = mem_pool_alloc_chunk (pool_p); - - memset (ptrs[j], 0, MEM_POOL_CHUNK_SIZE); - } - else - { - JERRY_ASSERT (j >= MEM_POOL_CHUNKS_NUMBER); - - ptrs[j] = NULL; - } - } - - // mem_heap_print (true); - - for (size_t j = 0; j < subiters; j++) - { - if (ptrs[j] != NULL) - { - for (size_t k = 0; k < MEM_POOL_CHUNK_SIZE; k++) - { - JERRY_ASSERT (((uint8_t*)ptrs[j])[k] == 0); - } - - mem_pool_free_chunk (pool_p, ptrs[j]); - } - } - } - - return 0; -} /* main */ diff --git a/tests/unit/test-poolman.cpp b/tests/unit/test-poolman.cpp index c7598e61a..d74fc03c3 100644 --- a/tests/unit/test-poolman.cpp +++ b/tests/unit/test-poolman.cpp @@ -20,16 +20,15 @@ #define JERRY_MEM_POOL_INTERNAL #include "mem-allocator.h" -#include "mem-pool.h" #include "mem-poolman.h" #include "test-common.h" // Iterations count -const uint32_t test_iters = 16384; +const uint32_t test_iters = 1024; // Subiterations count -const uint32_t test_max_sub_iters = 32; +const uint32_t test_max_sub_iters = 1024; uint8_t *ptrs[test_max_sub_iters]; @@ -91,5 +90,7 @@ main (int __attr_unused___ argc, stats.peak_allocated_chunks); #endif /* MEM_STATS */ + mem_finalize (false); + return 0; } /* main */