From 87750bea9da390bc99877a08f9a51017d0b73831 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 06:25:05 +0800 Subject: [PATCH 01/31] initial import of tlsf-bsd --- CHANGES | 63 +++ LICENSE | 25 ++ Makefile | 31 ++ README.md | 45 ++ tests/test-alloc.c | 106 +++++ tests/test-bits.c | 31 ++ tlsf/tlsf.c | 1028 ++++++++++++++++++++++++++++++++++++++++++++ tlsf/tlsf.h | 81 ++++ tlsf/tlsf_utils.h | 201 +++++++++ 9 files changed, 1611 insertions(+) create mode 100644 CHANGES create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 tests/test-alloc.c create mode 100644 tests/test-bits.c create mode 100644 tlsf/tlsf.c create mode 100644 tlsf/tlsf.h create mode 100644 tlsf/tlsf_utils.h diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..13fdd2b --- /dev/null +++ b/CHANGES @@ -0,0 +1,63 @@ +History +------- +2014/02/08 - v3.0 + * This version is based on improvements from 3DInteractive GmbH + * Interface changed to allow more than one memory pool + * Separated pool handling from control structure (adding, removing, debugging) + * Control structure and pools can still be constructed in the same memory block + * Memory blocks for control structure and pools are checked for alignment + * Added functions to retrieve control structure size, alignment size, min and + max block size, overhead of pool structure, and overhead of a single allocation + * Minimal Pool size is tlsf_block_size_min() + tlsf_pool_overhead() + * Pool must be empty when it is removed, in order to allow O(1) removal + +2011/10/20 - v2.0 + * 64-bit support + * More compiler intrinsics for ffs/fls + * ffs/fls verification during TLSF creation in debug builds + +2008/04/04 - v1.9 + * Add tlsf_heap_check, a heap integrity check + * Support a predefined tlsf_assert macro + * Fix realloc case where block should shrink; if adjacent block is + in use, execution would go down the slow path + +2007/02/08 - v1.8 + * Fix for unnecessary reallocation in tlsf_realloc + +2007/02/03 - v1.7 + * tlsf_heap_walk takes a callback + * tlsf_realloc now returns NULL on failure + * tlsf_memalign optimization for 4-byte alignment + * Usage of size_t where appropriate + +2006/11/21 - v1.6 + * ffs/fls broken out into tlsfbits.h + * tlsf_overhead queries per-pool overhead + +2006/11/07 - v1.5 + * Smart realloc implementation + * Smart memalign implementation + +2006/10/11 - v1.4 + * Add some ffs/fls implementations + * Minor code footprint reduction + +2006/09/14 - v1.3 + * Profiling indicates heavy use of blocks of + size 1-128, so implement small block handling + * Reduce pool overhead by about 1kb + * Reduce minimum block size from 32 to 12 bytes + * Realloc bug fix + +2006/09/09 - v1.2 + * Add tlsf_block_size + * Static assertion mechanism for invariants + * Minor bugfixes + +2006/09/01 - v1.1 + * Add tlsf_realloc + * Add tlsf_walk_heap + +2006/08/25 - v1.0 + * First release diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f886737 --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +/*- + * Copyright (c) 2006-2016, Matthew Conte + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..281c468 --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +OUT = build + +TARGETS = test-alloc test-bits +TARGETS := $(addprefix $(OUT)/,$(TARGETS)) + +all: $(TARGETS) + +CC = gcc +CFLAGS = -std=c99 -Wall -g -m32 -I tlsf + +OBJS = tlsf.o +OBJS := $(addprefix $(OUT)/,$(OBJS)) +deps := $(OBJS:%.o=%.o.d) + +$(OUT)/test-%: $(OBJS) tests/test-%.c + $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) + +$(OUT)/%.o: tlsf/%.c + @mkdir -p $(OUT) + $(CC) $(CFLAGS) -c -o $@ -MMD -MF $@.d $< + +CMDSEP = ; +check: $(TARGETS) + MALLOC_CHECK_=3 $(foreach prog,$(TARGETS),./$(prog) $(CMDSEP)) + +clean: + $(RM) $(TARGETS) $(OBJS) $(deps) + +.PHONY: all check clean + +-include $(deps) diff --git a/README.md b/README.md new file mode 100644 index 0000000..4d99e0e --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# TLSF-BSD + +Two Level Segregated Fit memory allocator implementation with O(1) +time complexity, licensed under the BSD License. + +Features +-------- +* O(1) cost for malloc, free, realloc, memalign +* Extremely low overhead per allocation (4 bytes) +* Low overhead per TLSF management of pools (~3kB) +* Low fragmentation +* Compiles to only a few kB of code and data +* Support for adding and removing memory pool regions on the fly + +Caveats +------- +* Currently, assumes architecture can make 4-byte aligned accesses +* Not designed to be thread safe; the user must provide this + +Notes +----- +This code was based on the TLSF 1.4 spec and documentation found at: + http://www.gii.upv.es/tlsf/main/docs + +This implementation was written to the specification of the document, +therefore no GPL restrictions apply. + +It also leverages the TLSF 2.0 improvement to shrink the per-block overhead +from 8 to 4 bytes. + +Known Issues +------------ +* Due to the internal block structure size and the implementation +details of tlsf_memalign, there is worst-case behavior when requesting +small (<16 byte) blocks aligned to 8-byte boundaries. Overuse of memalign +will generally increase fragmentation, but this particular case will leave +lots of unusable "holes" in the pool. The solution would be to internally +align all blocks to 8 bytes, but this will require significantl changes +to the implementation. + +Licensing +========= +TLSF-BSD is freely redistributable under the two-clause BSD License. +Use of this source code is governed by a BSD-style license that can be found +in the `LICENSE` file. diff --git a/tests/test-alloc.c b/tests/test-alloc.c new file mode 100644 index 0000000..26cba61 --- /dev/null +++ b/tests/test-alloc.c @@ -0,0 +1,106 @@ +#ifndef _BSD_SOURCE +#define _BSD_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static void random_test(const size_t spacelen, const size_t cap) +{ + const size_t overhead = tlsf_size() + tlsf_pool_overhead() + + tlsf_alloc_overhead(); + const size_t maxitems = spacelen; + size_t len; + uint8_t *space, *data; + unsigned i = 0; + tlsf_t *tlsf; + void **p; + + space = malloc(spacelen + overhead); + if (space == NULL) { + err(EXIT_FAILURE, "malloc"); + } + + p = malloc(maxitems * sizeof(void *)); + if (p == NULL) { + err(EXIT_FAILURE, "malloc"); + } + + tlsf = tlsf_create_with_pool(space, + spacelen + overhead); + assert(tlsf != NULL); + + /* + * Allocate random sizes up to the cap threshold. + * Track them in an array. + */ + for (;;) { + len = (random() % cap) + 1; + p[i] = tlsf_malloc(tlsf, len); + if (!p[i]) + break; + + /* Fill with magic (only when testing up to 1MB). */ + data = p[i]; + if (spacelen <= 1024 * 1024) { + memset(data, 0, len); + } + data[0] = 0xa5; + + if (i++ == maxitems) + break; + } + + /* + * Randomly deallocate the memory blocks until all of them are freed. + * The free space should match the free space after initialisation. + */ + for (unsigned n = i; n;) { + unsigned target = random() % i; + if (p[target] == NULL) + continue; + data = p[target]; + assert(data[0] == 0xa5); + tlsf_free(tlsf, p[target]); + p[target] = NULL; + n--; + } + + tlsf_destroy(tlsf); + free(space); + free(p); +} + +#define __arraycount(__x) \ + (sizeof(__x) / sizeof(__x[0])) + +static void random_sizes_test(void) +{ + const size_t sizes[] = {128, 1024, 1024 * 1024, 128 * 1024 * 1024}; + + for (unsigned i = 0; i < __arraycount(sizes); i++) { + unsigned n = 1024; + + while (n--) { + size_t cap = random() % sizes[i] + 1; + printf("sizes = %d, cap = %d\n", sizes[i], cap); + random_test(sizes[i], cap); + } + } +} + +int main(void) +{ + srandom(time(NULL) ^ getpid()); + random_sizes_test(); + puts("OK!"); + return 0; +} diff --git a/tests/test-bits.c b/tests/test-bits.c new file mode 100644 index 0000000..47dbbc0 --- /dev/null +++ b/tests/test-bits.c @@ -0,0 +1,31 @@ +#include +#include "tlsf_utils.h" + +int test_ffs_fls() +{ + /* Verify ffs/fls work properly. */ + int rv = 0; + rv += (tlsf_ffs(0) == -1) ? 0 : 0x1; + rv += (tlsf_fls(0) == -1) ? 0 : 0x2; + rv += (tlsf_ffs(1) == 0) ? 0 : 0x4; + rv += (tlsf_fls(1) == 0) ? 0 : 0x8; + rv += (tlsf_ffs(0x80000000) == 31) ? 0 : 0x10; + rv += (tlsf_ffs(0x80008000) == 15) ? 0 : 0x20; + rv += (tlsf_fls(0x80000008) == 31) ? 0 : 0x40; + rv += (tlsf_fls(0x7FFFFFFF) == 30) ? 0 : 0x80; + +#if defined(TLSF_64BIT) + rv += (tlsf_fls_sizet(0x80000000) == 31) ? 0 : 0x100; + rv += (tlsf_fls_sizet(0x100000000) == 32) ? 0 : 0x200; + rv += (tlsf_fls_sizet(0xffffffffffffffff) == 63) ? 0 : 0x400; +#endif + + if (rv) + printf("test_ffs_fls: %x ffs/fls tests failed.\n", rv); + return rv; +} + +int main() +{ + return test_ffs_fls(); +} diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c new file mode 100644 index 0000000..e28975c --- /dev/null +++ b/tlsf/tlsf.c @@ -0,0 +1,1028 @@ +/* COpyright (c) 2016 National Cheng Kung University. All rights reserved. + * Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include +#include +#include + +#include "tlsf.h" + +#include "tlsf_utils.h" + +#if __GNUC__ || __INTEL_COMPILER +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +/* Public constants: may be modified. */ +enum tlsf_public { + /* log2 of number of linear subdivisions of block sizes. Larger + * values require more memory in the control structure. Values of + * 4 or 5 are typical. + */ + SL_INDEX_COUNT_LOG2 = 5, +}; + +/* Private constants: do not modify. */ +enum tlsf_private { +#if defined(TLSF_64BIT) + /* All allocation sizes and addresses are aligned to 8 bytes. */ + ALIGN_SIZE_LOG2 = 3, +#else + /* All allocation sizes and addresses are aligned to 4 bytes. */ + ALIGN_SIZE_LOG2 = 2, +#endif + ALIGN_SIZE = (1 << ALIGN_SIZE_LOG2), + +/* + * We support allocations of sizes up to (1 << FL_INDEX_MAX) bits. + * However, because we linearly subdivide the second-level lists, and + * our minimum size granularity is 4 bytes, it doesn't make sense to + * create first-level lists for sizes smaller than SL_INDEX_COUNT * 4, + * or (1 << (SL_INDEX_COUNT_LOG2 + 2)) bytes, as there we will be + * trying to split size ranges into more slots than we have available. + * Instead, we calculate the minimum threshold size, and place all + * blocks below that size into the 0th first-level list. + */ + +#if defined(TLSF_64BIT) + /* + * TODO: We can increase this to support larger sizes, at the expense + * of more overhead in the TLSF structure. + */ + FL_INDEX_MAX = 32, +#else + FL_INDEX_MAX = 30, +#endif + SL_INDEX_COUNT = (1 << SL_INDEX_COUNT_LOG2), + FL_INDEX_SHIFT = (SL_INDEX_COUNT_LOG2 + ALIGN_SIZE_LOG2), + FL_INDEX_COUNT = (FL_INDEX_MAX - FL_INDEX_SHIFT + 1), + + SMALL_BLOCK_SIZE = (1 << FL_INDEX_SHIFT), +}; + +/* + * Cast and min/max macros. + */ +#define tlsf_cast(t, exp) ((t)(exp)) +#define tlsf_min(a, b) ((a) < (b) ? (a) : (b)) +#define tlsf_max(a, b) ((a) > (b) ? (a) : (b)) + +/* + * Set assert macro, if it has not been provided by the user. + */ +#if !defined(tlsf_assert) +#define tlsf_assert assert +#endif + +/* + * Static assertion mechanism. + */ + +#define _tlsf_glue2(x, y) x##y +#define _tlsf_glue(x, y) _tlsf_glue2(x, y) +#define tlsf_static_assert(exp) \ + typedef char _tlsf_glue(static_assert, __LINE__)[(exp) ? 1 : -1] + +/* This code has been tested on 32- and 64-bit (LP/LLP) architectures. */ +tlsf_static_assert(sizeof(int) * CHAR_BIT == 32); +tlsf_static_assert(sizeof(size_t) * CHAR_BIT >= 32); +tlsf_static_assert(sizeof(size_t) * CHAR_BIT <= 64); + +/* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */ +tlsf_static_assert(sizeof(unsigned int) * CHAR_BIT >= SL_INDEX_COUNT); + +/* Ensure we've properly tuned our sizes. */ +tlsf_static_assert(ALIGN_SIZE == SMALL_BLOCK_SIZE / SL_INDEX_COUNT); + +/* + * Data structures and associated constants. +*/ + +/* + * Block header structure. + * + * There are several implementation subtleties involved: + * - The prev_phys_block field is only valid if the previous block is free. + * - The prev_phys_block field is actually stored at the end of the + * previous block. It appears at the beginning of this structure only to + * simplify the implementation. + * - The next_free / prev_free fields are only valid if the block is free. + */ +typedef struct block_header_t { + /* Points to the previous physical block. */ + struct block_header_t *prev_phys_block; + + /* The size of this block, excluding the block header. */ + size_t size; + + /* Next and previous free blocks. */ + struct block_header_t *next_free; + struct block_header_t *prev_free; +} block_header_t; + +/* + * Since block sizes are always at least a multiple of 4, the two least + * significant bits of the size field are used to store the block status: + * - bit 0: whether block is busy or free + * - bit 1: whether previous block is busy or free + */ +static const size_t block_header_free_bit = 1 << 0; +static const size_t block_header_prev_free_bit = 1 << 1; + +/* + * The size of the block header exposed to used blocks is the size field. + * The prev_phys_block field is stored *inside* the previous free block. + */ +static const size_t block_header_overhead = sizeof(size_t); + +/* User data starts directly after the size field in a used block. */ +static const size_t block_start_offset = + offsetof(block_header_t, size) + sizeof(size_t); + +/* + * A free block must be large enough to store its header minus the size of + * the prev_phys_block field, and no larger than the number of addressable + * bits for FL_INDEX. + */ +static const size_t block_size_min = + sizeof(block_header_t) - sizeof(block_header_t *); +static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX; + +/* The TLSF control structure. */ +typedef struct control_t { + /* Empty lists point at this block to indicate they are free. */ + block_header_t block_null; + + /* Bitmaps for free lists. */ + unsigned int fl_bitmap; + unsigned int sl_bitmap[FL_INDEX_COUNT]; + + /* Head of free lists. */ + block_header_t *blocks[FL_INDEX_COUNT][SL_INDEX_COUNT]; +} control_t; + +/* A type used for casting when doing pointer arithmetic. */ +typedef ptrdiff_t tlsfptr_t; + +/* + * block_header_t member functions. + */ + +static size_t block_size(const block_header_t *block) +{ + return block->size & ~(block_header_free_bit | block_header_prev_free_bit); +} + +static void block_set_size(block_header_t *block, size_t size) +{ + const size_t oldsize = block->size; + block->size = + size | (oldsize & (block_header_free_bit | block_header_prev_free_bit)); +} + +static int block_is_last(const block_header_t *block) +{ + return block_size(block) == 0; +} + +static int block_is_free(const block_header_t *block) +{ + return tlsf_cast(int, block->size &block_header_free_bit); +} + +static void block_set_free(block_header_t *block) +{ + block->size |= block_header_free_bit; +} + +static void block_set_used(block_header_t *block) +{ + block->size &= ~block_header_free_bit; +} + +static int block_is_prev_free(const block_header_t *block) +{ + return tlsf_cast(int, block->size &block_header_prev_free_bit); +} + +static void block_set_prev_free(block_header_t *block) +{ + block->size |= block_header_prev_free_bit; +} + +static void block_set_prev_used(block_header_t *block) +{ + block->size &= ~block_header_prev_free_bit; +} + +static block_header_t *block_from_ptr(const void *ptr) +{ + return tlsf_cast(block_header_t *, + tlsf_cast(unsigned char *, ptr) - block_start_offset); +} + +static void *block_to_ptr(const block_header_t *block) +{ + return tlsf_cast(void *, + tlsf_cast(unsigned char *, block) + block_start_offset); +} + +/* Return location of next block after block of given size. */ +static block_header_t *offset_to_block(const void *ptr, size_t size) +{ + return tlsf_cast(block_header_t *, tlsf_cast(tlsfptr_t, ptr) + size); +} + +/* Return location of previous block. */ +static block_header_t *block_prev(const block_header_t *block) +{ + tlsf_assert(block_is_prev_free(block) && "previous block must be free"); + return block->prev_phys_block; +} + +/* Return location of next existing block. */ +static block_header_t *block_next(const block_header_t *block) +{ + block_header_t *next = offset_to_block( + block_to_ptr(block), block_size(block) - block_header_overhead); + tlsf_assert(!block_is_last(block)); + return next; +} + +/* Link a new block with its physical neighbor, return the neighbor. */ +static block_header_t *block_link_next(block_header_t *block) +{ + block_header_t *next = block_next(block); + next->prev_phys_block = block; + return next; +} + +static void block_mark_as_free(block_header_t *block) +{ + /* Link the block to the next block, first. */ + block_header_t *next = block_link_next(block); + block_set_prev_free(next); + block_set_free(block); +} + +static void block_mark_as_used(block_header_t *block) +{ + block_header_t *next = block_next(block); + block_set_prev_used(next); + block_set_used(block); +} + +static size_t align_up(size_t x, size_t align) +{ + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); + return (x + (align - 1)) & ~(align - 1); +} + +static size_t align_down(size_t x, size_t align) +{ + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); + return x - (x & (align - 1)); +} + +static void *align_ptr(const void *ptr, size_t align) +{ + const tlsfptr_t aligned = + (tlsf_cast(tlsfptr_t, ptr) + (align - 1)) & ~(align - 1); + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); + return tlsf_cast(void *, aligned); +} + +/* + * Adjust an allocation size to be aligned to word size, and no smaller + * than internal minimum. +*/ +static size_t adjust_request_size(size_t size, size_t align) +{ + size_t adjust = 0; + if (size && size < block_size_max) { + const size_t aligned = align_up(size, align); + adjust = tlsf_max(aligned, block_size_min); + } + return adjust; +} + +/* + * TLSF utility functions. In most cases, these are direct translations of + * the documentation found in the white paper. +*/ + +static void mapping_insert(size_t size, int *fli, int *sli) +{ + int fl, sl; + if (size < SMALL_BLOCK_SIZE) { + /* Store small blocks in first list. */ + fl = 0; + sl = tlsf_cast(int, size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT); + } else { + fl = tlsf_fls_sizet(size); + sl = tlsf_cast(int, size >> (fl - SL_INDEX_COUNT_LOG2)) ^ + (1 << SL_INDEX_COUNT_LOG2); + fl -= (FL_INDEX_SHIFT - 1); + } + *fli = fl; + *sli = sl; +} + +/* This version rounds up to the next block size (for allocations) */ +static void mapping_search(size_t size, int *fli, int *sli) +{ + if (size >= SMALL_BLOCK_SIZE) { + const size_t round = + (1 << (tlsf_fls_sizet(size) - SL_INDEX_COUNT_LOG2)) - 1; + size += round; + } + mapping_insert(size, fli, sli); +} + +static block_header_t *search_suitable_block(control_t *control, + int *fli, int *sli) +{ + int fl = *fli; + int sl = *sli; + + /* + * First, search for a block in the list associated with the given + * fl/sl index. + */ + unsigned int sl_map = control->sl_bitmap[fl] & (((unsigned int)~0) << sl); + if (!sl_map) { + /* No block exists. Search in the next largest first-level list. */ + const unsigned int fl_map = control->fl_bitmap & (((unsigned int)~0) << (fl + 1)); + if (!fl_map) { + /* No free blocks available, memory has been exhausted. */ + return 0; + } + + fl = tlsf_ffs(fl_map); + *fli = fl; + sl_map = control->sl_bitmap[fl]; + } + tlsf_assert(sl_map && "internal error - second level bitmap is null"); + sl = tlsf_ffs(sl_map); + *sli = sl; + + /* Return the first block in the free list. */ + return control->blocks[fl][sl]; +} + +/* Remove a free block from the free list.*/ +static void remove_free_block(control_t *control, + block_header_t *block, + int fl, int sl) +{ + block_header_t *prev = block->prev_free; + block_header_t *next = block->next_free; + tlsf_assert(prev && "prev_free field can not be null"); + tlsf_assert(next && "next_free field can not be null"); + next->prev_free = prev; + prev->next_free = next; + + /* If this block is the head of the free list, set new head. */ + if (control->blocks[fl][sl] == block) { + control->blocks[fl][sl] = next; + + /* If the new head is null, clear the bitmap. */ + if (next == &control->block_null) { + control->sl_bitmap[fl] &= ~(1 << sl); + + /* If the second bitmap is now empty, clear the fl bitmap. */ + if (!control->sl_bitmap[fl]) { + control->fl_bitmap &= ~(1 << fl); + } + } + } +} + +/* Insert a free block into the free block list. */ +static void insert_free_block(control_t *control, + block_header_t *block, + int fl, int sl) +{ + block_header_t *current = control->blocks[fl][sl]; + tlsf_assert(current && "free list cannot have a null entry"); + tlsf_assert(block && "cannot insert a null entry into the free list"); + block->next_free = current; + block->prev_free = &control->block_null; + current->prev_free = block; + + tlsf_assert(block_to_ptr(block) == + align_ptr(block_to_ptr(block), ALIGN_SIZE) && + "block not aligned properly"); + /* + * Insert the new block at the head of the list, and mark the first- + * and second-level bitmaps appropriately. + */ + control->blocks[fl][sl] = block; + control->fl_bitmap |= (1 << fl); + control->sl_bitmap[fl] |= (1 << sl); +} + +/* Remove a given block from the free list. */ +static void block_remove(control_t *control, block_header_t *block) +{ + int fl, sl; + mapping_insert(block_size(block), &fl, &sl); + remove_free_block(control, block, fl, sl); +} + +/* Insert a given block into the free list. */ +static void block_insert(control_t *control, block_header_t *block) +{ + int fl, sl; + mapping_insert(block_size(block), &fl, &sl); + insert_free_block(control, block, fl, sl); +} + +static int block_can_split(block_header_t *block, size_t size) +{ + return block_size(block) >= sizeof(block_header_t) + size; +} + +/* Split a block into two, the second of which is free. */ +static block_header_t *block_split(block_header_t *block, size_t size) +{ + /* Calculate the amount of space left in the remaining block. */ + block_header_t *remaining = offset_to_block(block_to_ptr(block), + size - block_header_overhead); + + const size_t remain_size = block_size(block) - + (size + block_header_overhead); + + tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), + ALIGN_SIZE) && + "remaining block not aligned properly"); + + tlsf_assert(block_size(block) == + remain_size + size + block_header_overhead); + block_set_size(remaining, remain_size); + tlsf_assert(block_size(remaining) >= block_size_min && + "block split with invalid size"); + + block_set_size(block, size); + block_mark_as_free(remaining); + + return remaining; +} + +/* Absorb a free block's storage into an adjacent previous free block. */ +static block_header_t *block_absorb(block_header_t *prev, block_header_t *block) +{ + tlsf_assert(!block_is_last(prev) && "previous block can't be last"); + /* Note: Leaves flags untouched. */ + prev->size += block_size(block) + block_header_overhead; + block_link_next(prev); + return prev; +} + +/* Merge a just-freed block with an adjacent previous free block. */ +static block_header_t *block_merge_prev(control_t *control, + block_header_t *block) +{ + if (block_is_prev_free(block)) { + block_header_t *prev = block_prev(block); + tlsf_assert(prev && "prev physical block can't be null"); + tlsf_assert(block_is_free(prev) && + "prev block is not free though marked as such"); + block_remove(control, prev); + block = block_absorb(prev, block); + } + + return block; +} + +/* Merge a just-freed block with an adjacent free block. */ +static block_header_t *block_merge_next(control_t *control, + block_header_t *block) +{ + block_header_t *next = block_next(block); + tlsf_assert(next && "next physical block can't be null"); + + if (block_is_free(next)) { + tlsf_assert(!block_is_last(block) && "previous block can't be last"); + block_remove(control, next); + block = block_absorb(block, next); + } + + return block; +} + +/* Trim any trailing block space off the end of a block, return to pool. */ +static void block_trim_free(control_t *control, + block_header_t *block, + size_t size) +{ + tlsf_assert(block_is_free(block) && "block must be free"); + if (block_can_split(block, size)) { + block_header_t *remaining_block = block_split(block, size); + block_link_next(block); + block_set_prev_free(remaining_block); + block_insert(control, remaining_block); + } +} + +/* Trim any trailing block space off the end of a used block, return to pool. */ +static void block_trim_used(control_t *control, + block_header_t *block, + size_t size) +{ + tlsf_assert(!block_is_free(block) && "block must be used"); + if (block_can_split(block, size)) { + /* If the next block is free, we must coalesce. */ + block_header_t *remaining_block = block_split(block, size); + block_set_prev_used(remaining_block); + + remaining_block = block_merge_next(control, remaining_block); + block_insert(control, remaining_block); + } +} + +static block_header_t *block_trim_free_leading(control_t *control, + block_header_t *block, + size_t size) +{ + block_header_t *remaining_block = block; + if (block_can_split(block, size)) { + /* We want the 2nd block. */ + remaining_block = block_split(block, size - block_header_overhead); + block_set_prev_free(remaining_block); + + block_link_next(block); + block_insert(control, block); + } + + return remaining_block; +} + +static block_header_t *block_locate_free(control_t *control, size_t size) +{ + int fl = 0, sl = 0; + block_header_t *block = 0; + + if (size) { + mapping_search(size, &fl, &sl); + block = search_suitable_block(control, &fl, &sl); + } + + if (block) { + tlsf_assert(block_size(block) >= size); + remove_free_block(control, block, fl, sl); + } + + return block; +} + +static void *block_prepare_used(control_t *control, + block_header_t *block, + size_t size) +{ + void *p = 0; + if (block) { + tlsf_assert(size && "size must be non-zero"); + block_trim_free(control, block, size); + block_mark_as_used(block); + p = block_to_ptr(block); + } + return p; +} + +/* Clear structure and point all empty lists at the null block. */ +static void control_construct(control_t *control) +{ + int i, j; + + control->block_null.next_free = &control->block_null; + control->block_null.prev_free = &control->block_null; + + control->fl_bitmap = 0; + for (i = 0; i < FL_INDEX_COUNT; ++i) { + control->sl_bitmap[i] = 0; + for (j = 0; j < SL_INDEX_COUNT; ++j) { + control->blocks[i][j] = &control->block_null; + } + } +} + +/* + * Debugging utilities. + */ + +typedef struct integrity_t { + int prev_status; + int status; +} integrity_t; + +#define tlsf_insist(x) \ + { \ + tlsf_assert(x); \ + if (!(x)) { \ + status--; \ + } \ + } + +static void integrity_walker(void *ptr, size_t size, int used, void *user) +{ + block_header_t *block = block_from_ptr(ptr); + integrity_t *integ = tlsf_cast(integrity_t *, user); + const int this_prev_status = block_is_prev_free(block) ? 1 : 0; + const int this_status = block_is_free(block) ? 1 : 0; + const size_t this_block_size = block_size(block); + + int status = 0; + (void)used; + tlsf_insist(integ->prev_status == this_prev_status && + "prev status incorrect"); + tlsf_insist(size == this_block_size && "block size incorrect"); + + integ->prev_status = this_status; + integ->status += status; +} + +int tlsf_check(tlsf_t tlsf) +{ + int i, j; + + control_t *control = tlsf_cast(control_t *, tlsf); + int status = 0; + + /* Check that the free lists and bitmaps are accurate. */ + for (i = 0; i < FL_INDEX_COUNT; ++i) { + for (j = 0; j < SL_INDEX_COUNT; ++j) { + const int fl_map = control->fl_bitmap & (1 << i); + const int sl_list = control->sl_bitmap[i]; + const int sl_map = sl_list & (1 << j); + const block_header_t *block = control->blocks[i][j]; + + /* Check that first- and second-level lists agree. */ + if (!fl_map) { + tlsf_insist(!sl_map && "second-level map must be null"); + } + + if (!sl_map) { + tlsf_insist(block == &control->block_null && + "block list must be null"); + continue; + } + + /* Check that there is at least one free block. */ + tlsf_insist(sl_list && "no free blocks in second-level map"); + tlsf_insist(block != &control->block_null && + "block should not be null"); + + while (block != &control->block_null) { + int fli, sli; + tlsf_insist(block_is_free(block) && "block should be free"); + tlsf_insist(!block_is_prev_free(block) && + "blocks should have coalesced"); + tlsf_insist(!block_is_free(block_next(block)) && + "blocks should have coalesced"); + tlsf_insist(block_is_prev_free(block_next(block)) && + "block should be free"); + tlsf_insist(block_size(block) >= block_size_min && + "block not minimum size"); + + mapping_insert(block_size(block), &fli, &sli); + tlsf_insist(fli == i && sli == j && + "block size indexed in wrong list"); + block = block->next_free; + } + } + } + + return status; +} + +#undef tlsf_insist + +static void default_walker(void *ptr, size_t size, int used, void *user) +{ + (void)user; + printf("\t%p %s size: %x (%p)\n", ptr, used ? "used" : "free", + (unsigned int)size, block_from_ptr(ptr)); +} + +void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void *user) +{ + tlsf_walker pool_walker = walker ? walker : default_walker; + block_header_t *block = offset_to_block(pool, -(int)block_header_overhead); + + while (block && !block_is_last(block)) { + pool_walker(block_to_ptr(block), block_size(block), + !block_is_free(block), user); + block = block_next(block); + } +} + +size_t tlsf_block_size(void *ptr) +{ + size_t size = 0; + if (ptr) { + const block_header_t *block = block_from_ptr(ptr); + size = block_size(block); + } + return size; +} + +int tlsf_check_pool(pool_t pool) +{ + /* Check that the blocks are physically correct. */ + integrity_t integ = {0, 0}; + tlsf_walk_pool(pool, integrity_walker, &integ); + + return integ.status; +} + +/* + * Size of the TLSF structures in a given memory block passed to + * tlsf_create, equal to the size of a control_t + */ +size_t tlsf_size(void) +{ + return sizeof(control_t); +} + +size_t tlsf_align_size(void) +{ + return ALIGN_SIZE; +} + +size_t tlsf_block_size_min(void) +{ + return block_size_min; +} + +size_t tlsf_block_size_max(void) +{ + return block_size_max; +} + +/* + * Overhead of the TLSF structures in a given memory block passed to + * tlsf_add_pool, equal to the overhead of a free block and the + * sentinel block. + */ +size_t tlsf_pool_overhead(void) +{ + return 2 * block_header_overhead; +} + +size_t tlsf_alloc_overhead(void) +{ + return block_header_overhead; +} + +pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes) +{ + block_header_t *block; + block_header_t *next; + + const size_t pool_overhead = tlsf_pool_overhead(); + const size_t pool_bytes = align_down(bytes - pool_overhead, ALIGN_SIZE); + + if (((ptrdiff_t)mem % ALIGN_SIZE) != 0) { + printf("tlsf_add_pool: Memory must be aligned by %u bytes.\n", + (unsigned int)ALIGN_SIZE); + return 0; + } + + if (pool_bytes < block_size_min || pool_bytes > block_size_max) { +#if defined(TLSF_64BIT) + printf( + "tlsf_add_pool: Memory size must be between 0x%x and 0x%x00 " + "bytes.\n", + (unsigned int)(pool_overhead + block_size_min), + (unsigned int)((pool_overhead + block_size_max) / 256)); +#else + printf("tlsf_add_pool: Memory size must be between %u and %u bytes.\n", + (unsigned int)(pool_overhead + block_size_min), + (unsigned int)(pool_overhead + block_size_max)); +#endif + return 0; + } + + /* + * Create the main free block. Offset the start of the block slightly + * so that the prev_phys_block field falls outside of the pool - + * it will never be used. + */ + block = offset_to_block(mem, -(tlsfptr_t)block_header_overhead); + block_set_size(block, pool_bytes); + block_set_free(block); + block_set_prev_used(block); + block_insert(tlsf_cast(control_t *, tlsf), block); + + /* Split the block to create a zero-size sentinel block. */ + next = block_link_next(block); + block_set_size(next, 0); + block_set_used(next); + block_set_prev_free(next); + + return mem; +} + +void tlsf_remove_pool(tlsf_t tlsf, pool_t pool) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + block_header_t *block = offset_to_block(pool, -(int)block_header_overhead); + + int fl = 0, sl = 0; + + tlsf_assert(block_is_free(block) && "block should be free"); + tlsf_assert(!block_is_free(block_next(block)) && + "next block should not be free"); + tlsf_assert(block_size(block_next(block)) == 0 && + "next block size should be zero"); + + mapping_insert(block_size(block), &fl, &sl); + remove_free_block(control, block, fl, sl); +} + +/* + * TLSF main interface. + */ + +tlsf_t tlsf_create(void *mem) +{ + if (((tlsfptr_t)mem % ALIGN_SIZE) != 0) { + printf("tlsf_create: Memory must be aligned to %u bytes.\n", + (unsigned int)ALIGN_SIZE); + return 0; + } + + control_construct(tlsf_cast(control_t *, mem)); + + return tlsf_cast(tlsf_t, mem); +} + +tlsf_t tlsf_create_with_pool(void *mem, size_t bytes) +{ + tlsf_t tlsf = tlsf_create(mem); + tlsf_add_pool(tlsf, (char *)mem + tlsf_size(), bytes - tlsf_size()); + return tlsf; +} + +void tlsf_destroy(tlsf_t tlsf) +{ + /* Nothing to do. */ + (void)tlsf; +} + +pool_t tlsf_get_pool(tlsf_t tlsf) +{ + return tlsf_cast(pool_t, (char *)tlsf + tlsf_size()); +} + +void *tlsf_malloc(tlsf_t tlsf, size_t size) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + block_header_t *block = block_locate_free(control, adjust); + return block_prepare_used(control, block, adjust); +} + +void *tlsf_memalign(tlsf_t tlsf, size_t align, size_t size) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + + /* + * We must allocate an additional minimum block size bytes so that if + * our free block will leave an alignment gap which is smaller, we can + * trim a leading free block and release it back to the pool. We must + * do this because the previous physical block is in use, therefore + * the prev_phys_block field is not valid, and we can't simply adjust + * the size of that block. + */ + const size_t gap_minimum = sizeof(block_header_t); + const size_t size_with_gap = + adjust_request_size(adjust + align + gap_minimum, align); + + /* + * If alignment is less than or equals base alignment, we're done. + * If we requested 0 bytes, return null, as tlsf_malloc(0) does. + */ + const size_t aligned_size = + (adjust && align > ALIGN_SIZE) ? size_with_gap : adjust; + + block_header_t *block = block_locate_free(control, aligned_size); + + /* This can't be a static assert. */ + tlsf_assert(sizeof(block_header_t) == + block_size_min + block_header_overhead); + + if (block) { + void *ptr = block_to_ptr(block); + void *aligned = align_ptr(ptr, align); + size_t gap = tlsf_cast( + size_t, tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr)); + + /* If gap size is too small, offset to next aligned boundary. */ + if (gap && gap < gap_minimum) { + const size_t gap_remain = gap_minimum - gap; + const size_t offset = tlsf_max(gap_remain, align); + const void *next_aligned = + tlsf_cast(void *, tlsf_cast(tlsfptr_t, aligned) + offset); + + aligned = align_ptr(next_aligned, align); + gap = tlsf_cast(size_t, tlsf_cast(tlsfptr_t, aligned) - + tlsf_cast(tlsfptr_t, ptr)); + } + + if (gap) { + tlsf_assert(gap >= gap_minimum && "gap size too small"); + block = block_trim_free_leading(control, block, gap); + } + } + + return block_prepare_used(control, block, adjust); +} + +void tlsf_free(tlsf_t tlsf, void *ptr) +{ + if (unlikely(!ptr)) return; + + control_t *control = tlsf_cast(control_t *, tlsf); + block_header_t *block = block_from_ptr(ptr); + tlsf_assert(!block_is_free(block) && "block already marked as free"); + block_mark_as_free(block); + block = block_merge_prev(control, block); + block = block_merge_next(control, block); + block_insert(control, block); +} + +/* + * The TLSF block information provides us with enough information to + * provide a reasonably intelligent implementation of realloc, growing or + * shrinking the currently allocated block as required. + * + * This routine handles the somewhat esoteric edge cases of realloc: + * - a non-zero size with a null pointer will behave like malloc + * - a zero size with a non-null pointer will behave like free + * - a request that cannot be satisfied will leave the original buffer + * untouched + * - an extended buffer size will leave the newly-allocated area with + * contents undefined + */ +void *tlsf_realloc(tlsf_t tlsf, void *ptr, size_t size) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + void *p = 0; + + /* Zero-size requests are treated as free. */ + if (ptr && size == 0) { + tlsf_free(tlsf, ptr); + } + /* Requests with NULL pointers are treated as malloc. */ + else if (!ptr) { + p = tlsf_malloc(tlsf, size); + } else { + block_header_t *block = block_from_ptr(ptr); + block_header_t *next = block_next(block); + + const size_t cursize = block_size(block); + const size_t combined = + cursize + block_size(next) + block_header_overhead; + const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + + tlsf_assert(!block_is_free(block) && "block already marked as free"); + + /* + * If the next block is used, or when combined with the current + * block, does not offer enough space, we must reallocate and copy. + */ + if (adjust > cursize && (!block_is_free(next) || adjust > combined)) { + p = tlsf_malloc(tlsf, size); + if (p) { + const size_t minsize = tlsf_min(cursize, size); + memcpy(p, ptr, minsize); + tlsf_free(tlsf, ptr); + } + } else { + /* Do we need to expand to the next block? */ + if (adjust > cursize) { + block_merge_next(control, block); + block_mark_as_used(block); + } + + /* Trim the resulting block and return the original pointer. */ + block_trim_used(control, block, adjust); + p = ptr; + } + } + + return p; +} diff --git a/tlsf/tlsf.h b/tlsf/tlsf.h new file mode 100644 index 0000000..afb3900 --- /dev/null +++ b/tlsf/tlsf.h @@ -0,0 +1,81 @@ +#ifndef INCLUDED_tlsf +#define INCLUDED_tlsf + +/* + * Two Level Segregated Fit memory allocator. + * + * Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the copyright holder nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL MATTHEW CONTE BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +/* tlsf_t: a TLSF structure. Can contain 1 to N pools. */ +/* pool_t: a block of memory that TLSF can manage. */ +typedef void *tlsf_t; +typedef void *pool_t; + +/* Create/destroy a memory pool. */ +tlsf_t tlsf_create(void *mem); +tlsf_t tlsf_create_with_pool(void *mem, size_t bytes); +void tlsf_destroy(tlsf_t tlsf); +pool_t tlsf_get_pool(tlsf_t tlsf); + +/* Add/remove memory pools. */ +pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes); +void tlsf_remove_pool(tlsf_t tlsf, pool_t pool); + +/* malloc/memalign/realloc/free replacements. */ +void *tlsf_malloc(tlsf_t tlsf, size_t bytes); +void *tlsf_memalign(tlsf_t tlsf, size_t align, size_t bytes); +void *tlsf_realloc(tlsf_t tlsf, void *ptr, size_t size); +void tlsf_free(tlsf_t tlsf, void *ptr); + +/* Returns internal block size, not original request size */ +size_t tlsf_block_size(void *ptr); + +/* Overheads/limits of internal structures. */ +size_t tlsf_size(void); +size_t tlsf_align_size(void); +size_t tlsf_block_size_min(void); +size_t tlsf_block_size_max(void); +size_t tlsf_pool_overhead(void); +size_t tlsf_alloc_overhead(void); + +/* Debugging. */ +typedef void (*tlsf_walker)(void *ptr, size_t size, int used, void *user); +void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void *user); +/* Returns nonzero if any internal consistency check fails. */ +int tlsf_check(tlsf_t tlsf); +int tlsf_check_pool(pool_t pool); + +#if defined(__cplusplus) +}; +#endif + +#endif diff --git a/tlsf/tlsf_utils.h b/tlsf/tlsf_utils.h new file mode 100644 index 0000000..da1d21c --- /dev/null +++ b/tlsf/tlsf_utils.h @@ -0,0 +1,201 @@ +#ifndef INCLUDED_tlsf_utils +#define INCLUDED_tlsf_utils + +#include + +/* +** Architecture-specific bit manipulation routines. +** +** TLSF achieves O(1) cost for malloc and free operations by limiting +** the search for a free block to a free list of guaranteed size +** adequate to fulfill the request, combined with efficient free list +** queries using bitmasks and architecture-specific bit-manipulation +** routines. +** +** Most modern processors provide instructions to count leading zeroes +** in a word, find the lowest and highest set bit, etc. These +** specific implementations will be used when available, falling back +** to a reasonably efficient generic implementation. +** +** NOTE: TLSF spec relies on ffs/fls returning value 0..31. +** ffs/fls return 1-32 by default, returning 0 for error. +*/ + +/* +** Detect whether or not we are building for a 32- or 64-bit (LP/LLP) +** architecture. There is no reliable portable method at compile-time. +*/ +#if defined(__alpha__) || defined(__ia64__) || defined(__x86_64__) || \ + defined(_WIN64) || defined(__LP64__) || defined(__LLP64__) +#define TLSF_64BIT +#endif + +#if defined(__GNUC__) + +/* count leading zeroes: + * clz(0) == 32, clz(0xf) == 28, clz(1 << 31) == 0 + */ +static inline int clz(uint32_t x) +{ + return x ? __builtin_clz(x) : sizeof(x) * 8; +} + +/* integer binary logarithm (rounding down): + * log2(0) == -1, log2(5) == 2 + */ +static inline int __log2(uint32_t x) +{ + return sizeof(x) * 8 - clz(x) - 1; +} + +/* find first set: + * __ffs(1) == 0, __ffs(0) == -1, __ffs(1<<31) == 31 + */ +static inline int tlsf_ffs(unsigned int word) +{ + return __log2(word & (uint32_t)(-(uint32_t) word)); +} + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = word ? 32 - __builtin_clz(word) : 0; + return bit - 1; +} + +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) && \ + (defined(_M_IX86) || defined(_M_X64)) +/* Microsoft Visual C++ support on x86/X64 architectures. */ + +#include + +#pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanForward) + +static inline int tlsf_fls(unsigned int word) +{ + unsigned long index; + return _BitScanReverse(&index, word) ? index : -1; +} + +static inline int tlsf_ffs(unsigned int word) +{ + unsigned long index; + return _BitScanForward(&index, word) ? index : -1; +} + +#elif defined(_MSC_VER) && defined(_M_PPC) +/* Microsoft Visual C++ support on PowerPC architectures. */ + +#include + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = 32 - _CountLeadingZeros(word); + return bit - 1; +} + +static inline int tlsf_ffs(unsigned int word) +{ + const unsigned int reverse = word & (~word + 1); + const int bit = 32 - _CountLeadingZeros(reverse); + return bit - 1; +} + +#elif defined(__ARMCC_VERSION) +/* RealView Compilation Tools for ARM */ + +static inline int tlsf_ffs(unsigned int word) +{ + const unsigned int reverse = word & (~word + 1); + const int bit = 32 - __clz(reverse); + return bit - 1; +} + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = word ? 32 - __clz(word) : 0; + return bit - 1; +} + +#elif defined(__ghs__) +/* Green Hills support for PowerPC */ + +#include + +static inline int tlsf_ffs(unsigned int word) +{ + const unsigned int reverse = word & (~word + 1); + const int bit = 32 - __CLZ32(reverse); + return bit - 1; +} + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = word ? 32 - __CLZ32(word) : 0; + return bit - 1; +} + +#else +/* Fall back to generic implementation. */ + +static inline int tlsf_fls_generic(unsigned int word) +{ + int bit = 32; + + if (!word) + bit -= 1; + if (!(word & 0xffff0000)) { + word <<= 16; + bit -= 16; + } + if (!(word & 0xff000000)) { + word <<= 8; + bit -= 8; + } + if (!(word & 0xf0000000)) { + word <<= 4; + bit -= 4; + } + if (!(word & 0xc0000000)) { + word <<= 2; + bit -= 2; + } + if (!(word & 0x80000000)) { + word <<= 1; + bit -= 1; + } + + return bit; +} + +/* Implement ffs in terms of fls. */ +static inline int tlsf_ffs(unsigned int word) +{ + return tlsf_fls_generic(word & (~word + 1)) - 1; +} + +static inline int tlsf_fls(unsigned int word) +{ + return tlsf_fls_generic(word) - 1; +} + +#endif + +/* Possibly 64-bit version of tlsf_fls. */ +#if defined(TLSF_64BIT) +static inline int tlsf_fls_sizet(size_t size) +{ + int high = (int)(size >> 32); + int bits = 0; + if (high) { + bits = 32 + tlsf_fls(high); + } else { + bits = tlsf_fls((int)size & 0xffffffff); + } + return bits; +} +#else +#define tlsf_fls_sizet tlsf_fls +#endif + +#endif /* INCLUDED_tlsf_utils */ From 948286b0a2d7ed143bb449d7fd82036c2c29da8a Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 06:28:12 +0800 Subject: [PATCH 02/31] add .gitignore --- .gitignore | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..81b995f --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +*~ +*.o +*.i +*.s +*.P +*.elf +*.bin +*.hex +*.map +*.swp +*.fuse* +*.pyc +*.swo +*.out +build/ +tags +core From 38b84b5fef5c6d037e6a8c64a6a33f80281d9ab3 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 06:28:53 +0800 Subject: [PATCH 03/31] add clang-format configurations --- .clang-format | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..2d07d20 --- /dev/null +++ b/.clang-format @@ -0,0 +1,12 @@ +BasedOnStyle: Chromium +Language: Cpp +MaxEmptyLinesToKeep: 3 +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +DerivePointerAlignment: false +PointerAlignment: Right +TabWidth: 4 +UseTab: Never +IndentWidth: 4 +BreakBeforeBraces: Linux +AccessModifierOffset: -4 From ca0f893f81e8f9a9e8cdd552b3fccefaaae0a030 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 06:31:54 +0800 Subject: [PATCH 04/31] clarify the copyright notice --- LICENSE | 49 +++++++++++++++++++++++++------------------------ tlsf/tlsf.c | 6 ++++-- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/LICENSE b/LICENSE index f886737..85fc745 100644 --- a/LICENSE +++ b/LICENSE @@ -1,25 +1,26 @@ -/*- - * Copyright (c) 2006-2016, Matthew Conte - * All rights reserved. +TLSF-BSD is freely redistributable under the two-clause BSD License: + +Copyright (c) 2016 National Cheng Kung University, Taiwan. +Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index e28975c..e5c7b6d 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -1,5 +1,7 @@ -/* COpyright (c) 2016 National Cheng Kung University. All rights reserved. - * Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. All rights reserved. +/* Copyright (c) 2016 National Cheng Kung University, Taiwan. + * Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. + * All rights reserved. + * * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. */ From fac6f76de5b0bb95fdc0fbe8cf886cccf9427b79 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 06:44:36 +0800 Subject: [PATCH 05/31] add reference to TLSF --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d99e0e..c498ab8 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # TLSF-BSD Two Level Segregated Fit memory allocator implementation with O(1) -time complexity, licensed under the BSD License. +time complexity, distributed under the BSD License. Features -------- @@ -22,6 +22,11 @@ Notes This code was based on the TLSF 1.4 spec and documentation found at: http://www.gii.upv.es/tlsf/main/docs +Reference: + M. Masmano, I. Ripoll, A. Crespo, and J. Real. + TLSF: a new dynamic memory allocator for real-time systems. + In Proc. ECRTS (2004), IEEE Computer Society, pp. 79-86. + This implementation was written to the specification of the document, therefore no GPL restrictions apply. From 680624b887cdffcfdce5ad27b4073f3fbe7608c0 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 07:18:49 +0800 Subject: [PATCH 06/31] make assertions configurable --- tlsf/tlsf.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index e5c7b6d..a25a53c 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -6,7 +6,6 @@ * found in the LICENSE file. */ -#include #include #include #include @@ -25,6 +24,13 @@ #define unlikely(x) (x) #endif +#ifdef TLSF_CONFIG_ASSERT +#include +#define tlsf_assert(expr) assert(expr) +#else +#define tlsf_assert(expr) (void)(0) +#endif + /* Public constants: may be modified. */ enum tlsf_public { /* log2 of number of linear subdivisions of block sizes. Larger @@ -79,17 +85,9 @@ enum tlsf_private { #define tlsf_min(a, b) ((a) < (b) ? (a) : (b)) #define tlsf_max(a, b) ((a) > (b) ? (a) : (b)) -/* - * Set assert macro, if it has not been provided by the user. - */ -#if !defined(tlsf_assert) -#define tlsf_assert assert -#endif - /* * Static assertion mechanism. */ - #define _tlsf_glue2(x, y) x##y #define _tlsf_glue(x, y) _tlsf_glue2(x, y) #define tlsf_static_assert(exp) \ @@ -108,7 +106,7 @@ tlsf_static_assert(ALIGN_SIZE == SMALL_BLOCK_SIZE / SL_INDEX_COUNT); /* * Data structures and associated constants. -*/ + */ /* * Block header structure. From d6698bfb3654c1c1bdc4c84f805d6741f4224273 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 07:23:19 +0800 Subject: [PATCH 07/31] wrap macro tlsf_insist with do { } while (0) --- tlsf/tlsf.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index a25a53c..5845789 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -627,12 +627,11 @@ typedef struct integrity_t { } integrity_t; #define tlsf_insist(x) \ - { \ + do { \ tlsf_assert(x); \ - if (!(x)) { \ + if (!(x)) \ status--; \ - } \ - } + } while (0) static void integrity_walker(void *ptr, size_t size, int used, void *user) { From d2945996b184cde8ab4b2057bcd5a0aeaf6ed88a Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 09:17:07 +0800 Subject: [PATCH 08/31] use NULL instead of 0 for returning NULL pointers --- tlsf/tlsf.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 5845789..0fc8110 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -365,7 +365,7 @@ static block_header_t *search_suitable_block(control_t *control, const unsigned int fl_map = control->fl_bitmap & (((unsigned int)~0) << (fl + 1)); if (!fl_map) { /* No free blocks available, memory has been exhausted. */ - return 0; + return NULL; } fl = tlsf_ffs(fl_map); @@ -571,7 +571,7 @@ static block_header_t *block_trim_free_leading(control_t *control, static block_header_t *block_locate_free(control_t *control, size_t size) { int fl = 0, sl = 0; - block_header_t *block = 0; + block_header_t *block = NULL; if (size) { mapping_search(size, &fl, &sl); @@ -590,7 +590,7 @@ static void *block_prepare_used(control_t *control, block_header_t *block, size_t size) { - void *p = 0; + void *p = NULL; if (block) { tlsf_assert(size && "size must be non-zero"); block_trim_free(control, block, size); @@ -859,7 +859,7 @@ tlsf_t tlsf_create(void *mem) if (((tlsfptr_t)mem % ALIGN_SIZE) != 0) { printf("tlsf_create: Memory must be aligned to %u bytes.\n", (unsigned int)ALIGN_SIZE); - return 0; + return NULL; } control_construct(tlsf_cast(control_t *, mem)); @@ -979,7 +979,7 @@ void tlsf_free(tlsf_t tlsf, void *ptr) void *tlsf_realloc(tlsf_t tlsf, void *ptr, size_t size) { control_t *control = tlsf_cast(control_t *, tlsf); - void *p = 0; + void *p = NULL; /* Zero-size requests are treated as free. */ if (ptr && size == 0) { From a6627a9ec49e17d83acfbcdb5c3bc369bbc83e96 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 09:17:55 +0800 Subject: [PATCH 09/31] drop includes We prefer to declare all types ourselves. --- tlsf/tlsf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 0fc8110..32adfad 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -7,7 +7,6 @@ */ #include -#include #include #include #include From 2fdcdbfbc00122f3fb41895c152d6adb27cb33f8 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 10:56:10 +0800 Subject: [PATCH 10/31] block_locate_free: fix potential edge case This may be fixing a bug generated by undefined behavior. --- tlsf/tlsf.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 32adfad..eeaed24 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -582,6 +582,9 @@ static block_header_t *block_locate_free(control_t *control, size_t size) remove_free_block(control, block, fl, sl); } + if (unlikely(block && !block->size)) + block = NULL; + return block; } From 445923a0929200c597a609df433e9307ced7cc8d Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 11:00:43 +0800 Subject: [PATCH 11/31] enable assertions by default build --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 281c468..d7b2e07 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ TARGETS := $(addprefix $(OUT)/,$(TARGETS)) all: $(TARGETS) CC = gcc -CFLAGS = -std=c99 -Wall -g -m32 -I tlsf +CFLAGS = -std=c99 -Wall -g -m32 -I tlsf \ + -D TLSF_CONFIG_ASSERT OBJS = tlsf.o OBJS := $(addprefix $(OUT)/,$(OBJS)) From 359cb2daad98400d5fad11e5bae21a228d5b2f3d Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 11:21:37 +0800 Subject: [PATCH 12/31] conditional build of debugging utilities --- tlsf/tlsf.c | 3 +++ tlsf/tlsf.h | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index eeaed24..78d3142 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -622,6 +622,7 @@ static void control_construct(control_t *control) /* * Debugging utilities. */ +#ifdef TSLF_CONFIG_DEBUG typedef struct integrity_t { int prev_status; @@ -747,6 +748,8 @@ int tlsf_check_pool(pool_t pool) return integ.status; } +#endif /* TLSF_CONFIG_DEBUG */ + /* * Size of the TLSF structures in a given memory block passed to * tlsf_create, equal to the size of a control_t diff --git a/tlsf/tlsf.h b/tlsf/tlsf.h index afb3900..49b8505 100644 --- a/tlsf/tlsf.h +++ b/tlsf/tlsf.h @@ -68,11 +68,13 @@ size_t tlsf_pool_overhead(void); size_t tlsf_alloc_overhead(void); /* Debugging. */ +#ifdef TSLF_CONFIG_DEBUG typedef void (*tlsf_walker)(void *ptr, size_t size, int used, void *user); void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void *user); /* Returns nonzero if any internal consistency check fails. */ int tlsf_check(tlsf_t tlsf); int tlsf_check_pool(pool_t pool); +#endif /* TSLF_CONFIG_DEBUG */ #if defined(__cplusplus) }; From de6c9982d6231d68d52a68a1ef90973d9061a5b5 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 11:29:40 +0800 Subject: [PATCH 13/31] tests: suppress 64-bit build warnings --- tests/test-alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test-alloc.c b/tests/test-alloc.c index 26cba61..7ca13f9 100644 --- a/tests/test-alloc.c +++ b/tests/test-alloc.c @@ -84,13 +84,13 @@ static void random_test(const size_t spacelen, const size_t cap) static void random_sizes_test(void) { - const size_t sizes[] = {128, 1024, 1024 * 1024, 128 * 1024 * 1024}; + const uint32_t sizes[] = {128, 1024, 1024 * 1024, 128 * 1024 * 1024}; for (unsigned i = 0; i < __arraycount(sizes); i++) { unsigned n = 1024; while (n--) { - size_t cap = random() % sizes[i] + 1; + uint32_t cap = random() % sizes[i] + 1; printf("sizes = %d, cap = %d\n", sizes[i], cap); random_test(sizes[i], cap); } From 80e1b01248ba92ab92bbbc048226769fe525ad3a Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 11:48:24 +0800 Subject: [PATCH 14/31] test bit manipulation before tlsf allocator itself --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d7b2e07..71b696c 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,16 @@ OUT = build -TARGETS = test-alloc test-bits +TARGETS = \ + test-bits \ + test-alloc TARGETS := $(addprefix $(OUT)/,$(TARGETS)) all: $(TARGETS) CC = gcc -CFLAGS = -std=c99 -Wall -g -m32 -I tlsf \ - -D TLSF_CONFIG_ASSERT +CFLAGS = \ + -std=c99 -Wall -g -m32 -I tlsf \ + -D TLSF_CONFIG_ASSERT OBJS = tlsf.o OBJS := $(addprefix $(OUT)/,$(OBJS)) From 4cc8c04c423e6ece56c470c9f9e8969fc664d371 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 13:41:12 +0800 Subject: [PATCH 15/31] defer to native bits configuration --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 71b696c..0258ce1 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ all: $(TARGETS) CC = gcc CFLAGS = \ - -std=c99 -Wall -g -m32 -I tlsf \ + -std=c99 -Wall -g -I tlsf \ -D TLSF_CONFIG_ASSERT OBJS = tlsf.o From f4b7123a90e61fc2d3d6b4bc3306162125bd70b0 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sun, 12 Jun 2016 14:32:59 +0800 Subject: [PATCH 16/31] tests: implement single-thread benchmark --- Makefile | 12 ++- tests/bench-alloc.c | 218 ++++++++++++++++++++++++++++++++++++++++++++ tests/lran2.h | 49 ++++++++++ 3 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 tests/bench-alloc.c create mode 100644 tests/lran2.h diff --git a/Makefile b/Makefile index 0258ce1..2807a16 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ OUT = build TARGETS = \ test-bits \ - test-alloc + test-alloc \ + bench-alloc TARGETS := $(addprefix $(OUT)/,$(TARGETS)) all: $(TARGETS) @@ -11,6 +12,7 @@ CC = gcc CFLAGS = \ -std=c99 -Wall -g -I tlsf \ -D TLSF_CONFIG_ASSERT +LDFLAGS = OBJS = tlsf.o OBJS := $(addprefix $(OUT)/,$(OBJS)) @@ -18,12 +20,18 @@ deps := $(OBJS:%.o=%.o.d) $(OUT)/test-%: $(OBJS) tests/test-%.c $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) +deps += build/test-bits.d + +$(OUT)/bench-%: $(OBJS) tests/bench-%.c + $(CC) $(CFLAGS) -o $@ -MMD -MF $@.d $^ $(LDFLAGS) +# FIXME: avoid hardcode +deps += build/bench-alloc.d $(OUT)/%.o: tlsf/%.c @mkdir -p $(OUT) $(CC) $(CFLAGS) -c -o $@ -MMD -MF $@.d $< -CMDSEP = ; +CMDSEP = ; echo "Please wait..." ; check: $(TARGETS) MALLOC_CHECK_=3 $(foreach prog,$(TARGETS),./$(prog) $(CMDSEP)) diff --git a/tests/bench-alloc.c b/tests/bench-alloc.c new file mode 100644 index 0000000..c360bdc --- /dev/null +++ b/tests/bench-alloc.c @@ -0,0 +1,218 @@ +/* Copyright (c) 2016 National Cheng Kung University, Taiwan. + * All rights reserved. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef _BSD_SOURCE +#define _BSD_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lran2.h" +#include "tlsf.h" + +static tlsf_t *tlsf; + +static void usage(const char *name) +{ + printf("run a malloc benchmark.\n" + "usage: %s [-s blk-size|blk-min:blk-max] [-l loop-count] " + "[-n num-blocks] [-c]\n", + name); + exit(-1); +} + +/* Parse an integer argument. */ +static int parse_int_arg(const char *arg, const char *exe_name) +{ + long int ret; + + errno = 0; + ret = strtol(arg, NULL, 0); + if (errno) + usage(exe_name); + + return (int) ret; +} + +/* Parse a size argument, which is either an integer or two integers + separated by a colon, denoting a range. */ +static void +parse_size_arg(const char *arg, const char *exe_name, + size_t *blk_min, size_t *blk_max) +{ + long int ret; + char *endptr; + + errno = 0; + ret = strtol(arg, &endptr, 0); + + if (errno) + usage(exe_name); + + *blk_min = (int)ret; + + if (endptr && *endptr == ':') { + errno = 0; + ret = strtol(endptr + 1, NULL, 0); + + if (errno) + usage(exe_name); + } + + *blk_max = (int)ret; + + if (blk_min > blk_max) + usage(exe_name); +} + +/* Get a random block size between blk_min and blk_max. */ +static size_t +get_random_block_size(size_t blk_min, size_t blk_max, + struct lran2_st *lran2_state) +{ + size_t blk_size; + + if (blk_max > blk_min) { + blk_size = blk_min + (lran2(lran2_state) % (blk_max - blk_min)); + } else + blk_size = blk_min; + + return blk_size; +} + +static void +run_alloc_benchmark(int loops, size_t blk_min, size_t blk_max, + void **blk_array, size_t num_blks, bool clear, + struct lran2_st *lran2_state) +{ + while (loops--) { + int next_idx = lran2(lran2_state) % num_blks; + size_t blk_size = get_random_block_size(blk_min, blk_max, lran2_state); + + if (blk_array[next_idx]) + tlsf_free(tlsf, blk_array[next_idx]); + + /* Insert the newly alloced block into the array at a random point. */ + blk_array[next_idx] = tlsf_malloc(tlsf, blk_size); + if (clear) + memset(blk_array[next_idx], 0, blk_size); + } + + /* Free up all allocated blocks. */ + for (size_t i = 0; i < num_blks; i++) { + if (blk_array[i]) + tlsf_free(tlsf, blk_array[i]); + } +} + +struct alloc_desc { + /* Generic fields. */ + int loops; + size_t blk_min; + size_t blk_max; + void **blk_array; + size_t num_blks; + bool clear; +}; + +static void start_bench(void *arg) +{ + struct alloc_desc *desc = arg; + struct lran2_st lran2_state; + + lran2_init(&lran2_state, time(NULL) ^ getpid()); + + run_alloc_benchmark(desc->loops, desc->blk_min, desc->blk_max, + desc->blk_array, desc->num_blks, desc->clear, + &lran2_state); +} + +int main(int argc, char **argv) +{ + size_t blk_min = 512, blk_max = 512, num_blks = 10000; + int loops = 10000000; + bool clear = false; + int opt; + + while ((opt = getopt(argc, argv, "s:l:r:t:n:b:ch")) > 0) { + switch (opt) { + case 's': + parse_size_arg(optarg, argv[0], &blk_min, &blk_max); + break; + case 'l': + loops = parse_int_arg(optarg, argv[0]); + break; + case 'n': + num_blks = parse_int_arg(optarg, argv[0]); + break; + case 'c': + clear = true; + break; + case 'h': + usage(argv[0]); + break; + default: + usage(argv[0]); + break; + } + } + + const size_t overhead = tlsf_size() + tlsf_pool_overhead() + + tlsf_alloc_overhead(); + char *space = malloc(blk_max * num_blks + overhead); + assert(space); + tlsf = tlsf_create_with_pool(space, blk_max * num_blks + overhead); + + struct alloc_desc desc = { + .loops = loops, + .blk_min = blk_min, + .blk_max = blk_max, + .blk_array = malloc(num_blks * sizeof(unsigned char *)), + .num_blks = num_blks, + .clear = clear, + }; + assert(desc.blk_array != NULL); + memset(desc.blk_array, 0, num_blks * sizeof(unsigned char *)); + + struct timespec start, end; + int err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start); + assert(err == 0); + + start_bench(&desc); + + err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end); + assert(err == 0); + + double elapsed = (end.tv_sec - start.tv_sec) + + (end.tv_nsec - start.tv_nsec) * 1e-9; + + struct rusage usage; + err = getrusage(RUSAGE_SELF, &usage); + assert(err == 0); + + tlsf_destroy(tlsf); + free(space); + + /* Dump both machine and human readable versions */ + printf("%u:%u:%u:%u:%u:%.6f: took %.6f s for %u malloc/free\n" + "benchmark loops of %u-%u bytes. ~%.3f us per loop\n", + blk_min, blk_max, loops, + (int)clear, usage.ru_maxrss, elapsed, elapsed, loops, blk_min, + blk_max, (double)(elapsed / loops) * 1e6); + + return 0; +} diff --git a/tests/lran2.h b/tests/lran2.h new file mode 100644 index 0000000..bf55a04 --- /dev/null +++ b/tests/lran2.h @@ -0,0 +1,49 @@ +/* Copyright (c) 1996 Wolfram Gloger. + * A small, portable pseudo-random number generator. + */ + +#ifndef _LRAN2_H +#define _LRAN2_H + +#define LRAN2_MAX 714025l /* constants for portable */ +#define IA 1366l /* random number generator */ +#define IC 150889l /* (see e.g. `Numerical Recipes') */ + +struct lran2_st { + long x, y, v[97]; +}; + +static void lran2_init(struct lran2_st *d, long seed) +{ + long x; + int j; + + x = (IC - seed) % LRAN2_MAX; + if (x < 0) + x = -x; + for (j = 0; j < 97; j++) { + x = (IA * x + IC) % LRAN2_MAX; + d->v[j] = x; + } + d->x = (IA * x + IC) % LRAN2_MAX; + d->y = d->x; +} + +#ifdef __GNUC__ +__inline__ +#endif + static long + lran2(struct lran2_st *d) +{ + int j = (d->y % 97); + + d->y = d->v[j]; + d->x = (IA * d->x + IC) % LRAN2_MAX; + d->v[j] = d->x; + return d->y; +} + +#undef IA +#undef IC + +#endif From 42c148d17881c7e57f8d7c52aedbf33cf5f4be10 Mon Sep 17 00:00:00 2001 From: ldotrg Date: Thu, 16 Jun 2016 23:21:20 +0800 Subject: [PATCH 17/31] Add _POSIX_C_SOURCE macro, to make sure the preprocessor was getting the library features correctly --- tests/bench-alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bench-alloc.c b/tests/bench-alloc.c index c360bdc..1d9d050 100644 --- a/tests/bench-alloc.c +++ b/tests/bench-alloc.c @@ -8,7 +8,7 @@ #ifndef _BSD_SOURCE #define _BSD_SOURCE #endif - +#define _POSIX_C_SOURCE 199309L #include #include #include From a3f9bc0a75415e884f1180d543816ad559469283 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 29 Nov 2016 15:03:23 +0800 Subject: [PATCH 18/31] adjust_request_size: 64-bit friendly An allocation of request in the range [block_size_max - ALIGN_SIZE, block_size_max] could cause an out-of-bounds access to sl_bitmap. --- tlsf/tlsf.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 78d3142..7d02d1c 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -308,9 +308,13 @@ static void *align_ptr(const void *ptr, size_t align) static size_t adjust_request_size(size_t size, size_t align) { size_t adjust = 0; - if (size && size < block_size_max) { + if (size) { const size_t aligned = align_up(size, align); - adjust = tlsf_max(aligned, block_size_min); + + /* aligned sized must not exceed block_size_max */ + if (aligned < block_size_max) { + adjust = tlsf_max(aligned, block_size_min); + } } return adjust; } From d60fbf97a74c8a80d59393532e2a4e17b64f2205 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 29 Nov 2016 15:05:27 +0800 Subject: [PATCH 19/31] avoid negative size_t --- tlsf/tlsf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 7d02d1c..2399999 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -237,7 +237,7 @@ static void *block_to_ptr(const block_header_t *block) } /* Return location of next block after block of given size. */ -static block_header_t *offset_to_block(const void *ptr, size_t size) +static block_header_t *offset_to_block(const void *ptr, tlsfptr_t size) { return tlsf_cast(block_header_t *, tlsf_cast(tlsfptr_t, ptr) + size); } From 6c16aaa94e56c2d1e776fb3d66bc56ea6d619b71 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 29 Nov 2016 15:11:59 +0800 Subject: [PATCH 20/31] lran2: enforce static inline for available functions --- tests/lran2.h | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/lran2.h b/tests/lran2.h index bf55a04..34acdc8 100644 --- a/tests/lran2.h +++ b/tests/lran2.h @@ -13,15 +13,13 @@ struct lran2_st { long x, y, v[97]; }; -static void lran2_init(struct lran2_st *d, long seed) +static inline +void lran2_init(struct lran2_st *d, long seed) { - long x; - int j; - - x = (IC - seed) % LRAN2_MAX; + long x = (IC - seed) % LRAN2_MAX; if (x < 0) x = -x; - for (j = 0; j < 97; j++) { + for (int j = 0; j < 97; j++) { x = (IA * x + IC) % LRAN2_MAX; d->v[j] = x; } @@ -29,11 +27,8 @@ static void lran2_init(struct lran2_st *d, long seed) d->y = d->x; } -#ifdef __GNUC__ -__inline__ -#endif - static long - lran2(struct lran2_st *d) +static inline +long lran2(struct lran2_st *d) { int j = (d->y % 97); From 5ee78e9c12cb378cbf3d42211953c48c47f3d08c Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 29 Nov 2016 15:18:34 +0800 Subject: [PATCH 21/31] tests: eliminate memory leaks --- tests/bench-alloc.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/bench-alloc.c b/tests/bench-alloc.c index 1d9d050..0feb056 100644 --- a/tests/bench-alloc.c +++ b/tests/bench-alloc.c @@ -141,6 +141,13 @@ static void start_bench(void *arg) &lran2_state); } +static void stop_bench(void *arg) +{ + struct alloc_desc *desc = arg; + if (!desc) return; + free(desc->blk_array); +} + int main(int argc, char **argv) { size_t blk_min = 512, blk_max = 512, num_blks = 10000; @@ -189,13 +196,14 @@ int main(int argc, char **argv) memset(desc.blk_array, 0, num_blks * sizeof(unsigned char *)); struct timespec start, end; + int err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start); assert(err == 0); - start_bench(&desc); err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end); assert(err == 0); + stop_bench(&desc); double elapsed = (end.tv_sec - start.tv_sec) + (end.tv_nsec - start.tv_nsec) * 1e-9; From 707b788531becad0a31efbf721d18ac74b556486 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 29 Nov 2016 15:24:49 +0800 Subject: [PATCH 22/31] _BSD_SOURCE is deprecated, use _DEFAULT_SOURCE --- tests/bench-alloc.c | 5 +++-- tests/test-alloc.c | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/bench-alloc.c b/tests/bench-alloc.c index 0feb056..3d35ca5 100644 --- a/tests/bench-alloc.c +++ b/tests/bench-alloc.c @@ -5,10 +5,11 @@ * found in the LICENSE file. */ -#ifndef _BSD_SOURCE -#define _BSD_SOURCE +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE #endif #define _POSIX_C_SOURCE 199309L + #include #include #include diff --git a/tests/test-alloc.c b/tests/test-alloc.c index 7ca13f9..c0d704f 100644 --- a/tests/test-alloc.c +++ b/tests/test-alloc.c @@ -1,5 +1,5 @@ -#ifndef _BSD_SOURCE -#define _BSD_SOURCE +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE #endif #include From 39c90a656d1d29dbc98a2488ddf7b30a60153907 Mon Sep 17 00:00:00 2001 From: Vitor Menezes Date: Wed, 21 Mar 2018 00:06:43 +0800 Subject: [PATCH 23/31] Protect against large sizes resulting in off-the-end free blocks Signed-off-by: Jim Huang --- tlsf/tlsf.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 2399999..cea6c83 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -578,7 +578,18 @@ static block_header_t *block_locate_free(control_t *control, size_t size) if (size) { mapping_search(size, &fl, &sl); - block = search_suitable_block(control, &fl, &sl); + /* + * mapping_search can futz with the size, so for excessively large + * sizes it can sometimes wind up with indices that are off the end + * of the block array. + * So, we protect against that here, since this is the only callsite of + * mapping_search. + * Note that we don't need to check sl, since it comes from a modulo + * operation that guarantees it's always in range. + */ + if (fl < FL_INDEX_COUNT) { + block = search_suitable_block(control, &fl, &sl); + } } if (block) { From d6dc1228f5434e1be880efaeb0484b05b61a9f09 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 21 Mar 2018 00:33:44 +0800 Subject: [PATCH 24/31] preliminary AArch64 support --- tlsf/tlsf_utils.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tlsf/tlsf_utils.h b/tlsf/tlsf_utils.h index da1d21c..79d71b3 100644 --- a/tlsf/tlsf_utils.h +++ b/tlsf/tlsf_utils.h @@ -26,7 +26,8 @@ ** architecture. There is no reliable portable method at compile-time. */ #if defined(__alpha__) || defined(__ia64__) || defined(__x86_64__) || \ - defined(_WIN64) || defined(__LP64__) || defined(__LLP64__) + defined(_WIN64) || defined(__LP64__) || defined(__LLP64__) || \ + defined(__aarch64__) #define TLSF_64BIT #endif From 82fa887f714b79c6545a89c6a835d20512929a5a Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Mon, 29 Oct 2018 21:12:03 +0800 Subject: [PATCH 25/31] Fix typo in debug option Close #2 --- tlsf/tlsf.c | 2 +- tlsf/tlsf.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index cea6c83..de498a8 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -637,7 +637,7 @@ static void control_construct(control_t *control) /* * Debugging utilities. */ -#ifdef TSLF_CONFIG_DEBUG +#ifdef TLSF_CONFIG_DEBUG typedef struct integrity_t { int prev_status; diff --git a/tlsf/tlsf.h b/tlsf/tlsf.h index 49b8505..74a6b56 100644 --- a/tlsf/tlsf.h +++ b/tlsf/tlsf.h @@ -68,13 +68,13 @@ size_t tlsf_pool_overhead(void); size_t tlsf_alloc_overhead(void); /* Debugging. */ -#ifdef TSLF_CONFIG_DEBUG +#ifdef TLSF_CONFIG_DEBUG typedef void (*tlsf_walker)(void *ptr, size_t size, int used, void *user); void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void *user); /* Returns nonzero if any internal consistency check fails. */ int tlsf_check(tlsf_t tlsf); int tlsf_check_pool(pool_t pool); -#endif /* TSLF_CONFIG_DEBUG */ +#endif /* TLSF_CONFIG_DEBUG */ #if defined(__cplusplus) }; From fc723e8bebecf7add4d41582bf0ae82eb83ba1b2 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Mon, 29 Oct 2018 23:03:25 +0800 Subject: [PATCH 26/31] Shifting of negative values is undefined in C99 --- tlsf/tlsf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index de498a8..615010d 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -401,11 +401,11 @@ static void remove_free_block(control_t *control, /* If the new head is null, clear the bitmap. */ if (next == &control->block_null) { - control->sl_bitmap[fl] &= ~(1 << sl); + control->sl_bitmap[fl] &= ~(1U << sl); /* If the second bitmap is now empty, clear the fl bitmap. */ if (!control->sl_bitmap[fl]) { - control->fl_bitmap &= ~(1 << fl); + control->fl_bitmap &= ~(1U << fl); } } } @@ -431,8 +431,8 @@ static void insert_free_block(control_t *control, * and second-level bitmaps appropriately. */ control->blocks[fl][sl] = block; - control->fl_bitmap |= (1 << fl); - control->sl_bitmap[fl] |= (1 << sl); + control->fl_bitmap |= (1U << fl); + control->sl_bitmap[fl] |= (1U << sl); } /* Remove a given block from the free list. */ From 20f72990cb42759e2898cb08ebf2f50e7acd4a54 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Mon, 29 Oct 2018 23:53:16 +0800 Subject: [PATCH 27/31] Do pointer arithmetic directly --- tlsf/tlsf.c | 50 ++++++++++++++++++++++++----------------------- tlsf/tlsf_utils.h | 6 +++--- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 615010d..81885c6 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -170,9 +170,6 @@ typedef struct control_t { block_header_t *blocks[FL_INDEX_COUNT][SL_INDEX_COUNT]; } control_t; -/* A type used for casting when doing pointer arithmetic. */ -typedef ptrdiff_t tlsfptr_t; - /* * block_header_t member functions. */ @@ -237,9 +234,9 @@ static void *block_to_ptr(const block_header_t *block) } /* Return location of next block after block of given size. */ -static block_header_t *offset_to_block(const void *ptr, tlsfptr_t size) +static block_header_t *offset_to_block(const void *ptr, ptrdiff_t size) { - return tlsf_cast(block_header_t *, tlsf_cast(tlsfptr_t, ptr) + size); + return tlsf_cast(block_header_t *, tlsf_cast(ptrdiff_t, ptr) + size); } /* Return location of previous block. */ @@ -295,8 +292,8 @@ static size_t align_down(size_t x, size_t align) static void *align_ptr(const void *ptr, size_t align) { - const tlsfptr_t aligned = - (tlsf_cast(tlsfptr_t, ptr) + (align - 1)) & ~(align - 1); + const ptrdiff_t aligned = + (tlsf_cast(ptrdiff_t, ptr) + (align - 1)) & ~(align - 1); tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); return tlsf_cast(void *, aligned); } @@ -353,7 +350,8 @@ static void mapping_search(size_t size, int *fli, int *sli) } static block_header_t *search_suitable_block(control_t *control, - int *fli, int *sli) + int *fli, + int *sli) { int fl = *fli; int sl = *sli; @@ -365,7 +363,8 @@ static block_header_t *search_suitable_block(control_t *control, unsigned int sl_map = control->sl_bitmap[fl] & (((unsigned int)~0) << sl); if (!sl_map) { /* No block exists. Search in the next largest first-level list. */ - const unsigned int fl_map = control->fl_bitmap & (((unsigned int)~0) << (fl + 1)); + const unsigned int fl_map = + control->fl_bitmap & (((unsigned int)~0) << (fl + 1)); if (!fl_map) { /* No free blocks available, memory has been exhausted. */ return NULL; @@ -386,7 +385,8 @@ static block_header_t *search_suitable_block(control_t *control, /* Remove a free block from the free list.*/ static void remove_free_block(control_t *control, block_header_t *block, - int fl, int sl) + int fl, + int sl) { block_header_t *prev = block->prev_free; block_header_t *next = block->next_free; @@ -414,7 +414,8 @@ static void remove_free_block(control_t *control, /* Insert a free block into the free block list. */ static void insert_free_block(control_t *control, block_header_t *block, - int fl, int sl) + int fl, + int sl) { block_header_t *current = control->blocks[fl][sl]; tlsf_assert(current && "free list cannot have a null entry"); @@ -460,14 +461,14 @@ static int block_can_split(block_header_t *block, size_t size) static block_header_t *block_split(block_header_t *block, size_t size) { /* Calculate the amount of space left in the remaining block. */ - block_header_t *remaining = offset_to_block(block_to_ptr(block), - size - block_header_overhead); + block_header_t *remaining = + offset_to_block(block_to_ptr(block), size - block_header_overhead); - const size_t remain_size = block_size(block) - - (size + block_header_overhead); + const size_t remain_size = + block_size(block) - (size + block_header_overhead); - tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), - ALIGN_SIZE) && + tlsf_assert(block_to_ptr(remaining) == + align_ptr(block_to_ptr(remaining), ALIGN_SIZE) && "remaining block not aligned properly"); tlsf_assert(block_size(block) == @@ -838,7 +839,7 @@ pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes) * so that the prev_phys_block field falls outside of the pool - * it will never be used. */ - block = offset_to_block(mem, -(tlsfptr_t)block_header_overhead); + block = offset_to_block(mem, -(ptrdiff_t)block_header_overhead); block_set_size(block, pool_bytes); block_set_free(block); block_set_prev_used(block); @@ -876,7 +877,7 @@ void tlsf_remove_pool(tlsf_t tlsf, pool_t pool) tlsf_t tlsf_create(void *mem) { - if (((tlsfptr_t)mem % ALIGN_SIZE) != 0) { + if (((ptrdiff_t)mem % ALIGN_SIZE) != 0) { printf("tlsf_create: Memory must be aligned to %u bytes.\n", (unsigned int)ALIGN_SIZE); return NULL; @@ -947,18 +948,18 @@ void *tlsf_memalign(tlsf_t tlsf, size_t align, size_t size) void *ptr = block_to_ptr(block); void *aligned = align_ptr(ptr, align); size_t gap = tlsf_cast( - size_t, tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr)); + size_t, tlsf_cast(ptrdiff_t, aligned) - tlsf_cast(ptrdiff_t, ptr)); /* If gap size is too small, offset to next aligned boundary. */ if (gap && gap < gap_minimum) { const size_t gap_remain = gap_minimum - gap; const size_t offset = tlsf_max(gap_remain, align); const void *next_aligned = - tlsf_cast(void *, tlsf_cast(tlsfptr_t, aligned) + offset); + tlsf_cast(void *, tlsf_cast(ptrdiff_t, aligned) + offset); aligned = align_ptr(next_aligned, align); - gap = tlsf_cast(size_t, tlsf_cast(tlsfptr_t, aligned) - - tlsf_cast(tlsfptr_t, ptr)); + gap = tlsf_cast(size_t, tlsf_cast(ptrdiff_t, aligned) - + tlsf_cast(ptrdiff_t, ptr)); } if (gap) { @@ -972,7 +973,8 @@ void *tlsf_memalign(tlsf_t tlsf, size_t align, size_t size) void tlsf_free(tlsf_t tlsf, void *ptr) { - if (unlikely(!ptr)) return; + if (unlikely(!ptr)) + return; control_t *control = tlsf_cast(control_t *, tlsf); block_header_t *block = block_from_ptr(ptr); diff --git a/tlsf/tlsf_utils.h b/tlsf/tlsf_utils.h index 79d71b3..144e07f 100644 --- a/tlsf/tlsf_utils.h +++ b/tlsf/tlsf_utils.h @@ -26,7 +26,7 @@ ** architecture. There is no reliable portable method at compile-time. */ #if defined(__alpha__) || defined(__ia64__) || defined(__x86_64__) || \ - defined(_WIN64) || defined(__LP64__) || defined(__LLP64__) || \ + defined(_WIN64) || defined(__LP64__) || defined(__LLP64__) || \ defined(__aarch64__) #define TLSF_64BIT #endif @@ -54,7 +54,7 @@ static inline int __log2(uint32_t x) */ static inline int tlsf_ffs(unsigned int word) { - return __log2(word & (uint32_t)(-(uint32_t) word)); + return __log2(word & (uint32_t)(-(uint32_t)word)); } static inline int tlsf_fls(unsigned int word) @@ -64,7 +64,7 @@ static inline int tlsf_fls(unsigned int word) } #elif defined(_MSC_VER) && (_MSC_VER >= 1400) && \ - (defined(_M_IX86) || defined(_M_X64)) + (defined(_M_IX86) || defined(_M_X64)) /* Microsoft Visual C++ support on x86/X64 architectures. */ #include From 859ace5c2577d25055db1875bd40e81db6bf8217 Mon Sep 17 00:00:00 2001 From: Periklis Akritidis Date: Tue, 30 Oct 2018 00:10:31 +0800 Subject: [PATCH 28/31] Consolidate metadata size Fixed block_size_min to be sizeof(block_header_t) minus the size of metadata instead of the size of the prev_phys_block pointer. These happen to be of the same size, but it is conceptually wrong. For example, we may want to expand the metadata per allocation For convenience, also changed offset_to_block() to take a block size instead of block size - block_header_overhead Signed-off-by: Jim Huang --- tlsf/tlsf.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index 81885c6..cd577b0 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -144,17 +144,23 @@ static const size_t block_header_prev_free_bit = 1 << 1; */ static const size_t block_header_overhead = sizeof(size_t); +/* + * The size of the block header that overlaps the previous block, + * namely the size of prev_phys_block field. + */ +static const size_t block_header_overlap = sizeof(block_header_t *); + /* User data starts directly after the size field in a used block. */ static const size_t block_start_offset = offsetof(block_header_t, size) + sizeof(size_t); /* * A free block must be large enough to store its header minus the size of - * the prev_phys_block field, and no larger than the number of addressable + * the metadata field, and no larger than the number of addressable * bits for FL_INDEX. */ static const size_t block_size_min = - sizeof(block_header_t) - sizeof(block_header_t *); + sizeof(block_header_t) - sizeof(size_t); // FIXME: metadata static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX; /* The TLSF control structure. */ @@ -236,7 +242,8 @@ static void *block_to_ptr(const block_header_t *block) /* Return location of next block after block of given size. */ static block_header_t *offset_to_block(const void *ptr, ptrdiff_t size) { - return tlsf_cast(block_header_t *, tlsf_cast(ptrdiff_t, ptr) + size); + return tlsf_cast(block_header_t *, + tlsf_cast(ptrdiff_t, ptr) + size - block_header_overlap); } /* Return location of previous block. */ @@ -249,8 +256,8 @@ static block_header_t *block_prev(const block_header_t *block) /* Return location of next existing block. */ static block_header_t *block_next(const block_header_t *block) { - block_header_t *next = offset_to_block( - block_to_ptr(block), block_size(block) - block_header_overhead); + block_header_t *next = + offset_to_block(block_to_ptr(block), block_size(block)); tlsf_assert(!block_is_last(block)); return next; } @@ -461,8 +468,7 @@ static int block_can_split(block_header_t *block, size_t size) static block_header_t *block_split(block_header_t *block, size_t size) { /* Calculate the amount of space left in the remaining block. */ - block_header_t *remaining = - offset_to_block(block_to_ptr(block), size - block_header_overhead); + block_header_t *remaining = offset_to_block(block_to_ptr(block), size); const size_t remain_size = block_size(block) - (size + block_header_overhead); @@ -555,6 +561,8 @@ static void block_trim_used(control_t *control, } } +/* If possible, create a trailing free block after trimming given block by size + */ static block_header_t *block_trim_free_leading(control_t *control, block_header_t *block, size_t size) @@ -736,7 +744,7 @@ static void default_walker(void *ptr, size_t size, int used, void *user) void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void *user) { tlsf_walker pool_walker = walker ? walker : default_walker; - block_header_t *block = offset_to_block(pool, -(int)block_header_overhead); + block_header_t *block = offset_to_block(pool, 0); while (block && !block_is_last(block)) { pool_walker(block_to_ptr(block), block_size(block), @@ -839,7 +847,7 @@ pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes) * so that the prev_phys_block field falls outside of the pool - * it will never be used. */ - block = offset_to_block(mem, -(ptrdiff_t)block_header_overhead); + block = offset_to_block(mem, 0); block_set_size(block, pool_bytes); block_set_free(block); block_set_prev_used(block); @@ -857,7 +865,7 @@ pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes) void tlsf_remove_pool(tlsf_t tlsf, pool_t pool) { control_t *control = tlsf_cast(control_t *, tlsf); - block_header_t *block = offset_to_block(pool, -(int)block_header_overhead); + block_header_t *block = offset_to_block(pool, 0); int fl = 0, sl = 0; From 7ee139758752a1feaace91c21b9af7d248f72c8a Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 30 Oct 2018 00:20:24 +0800 Subject: [PATCH 29/31] Use %zu to eliminate conditional build Format specifier %zu was added in C99. --- tlsf/tlsf.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index cd577b0..cc1c555 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -828,17 +828,9 @@ pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes) } if (pool_bytes < block_size_min || pool_bytes > block_size_max) { -#if defined(TLSF_64BIT) printf( - "tlsf_add_pool: Memory size must be between 0x%x and 0x%x00 " - "bytes.\n", - (unsigned int)(pool_overhead + block_size_min), - (unsigned int)((pool_overhead + block_size_max) / 256)); -#else - printf("tlsf_add_pool: Memory size must be between %u and %u bytes.\n", - (unsigned int)(pool_overhead + block_size_min), - (unsigned int)(pool_overhead + block_size_max)); -#endif + "tlsf_add_pool: Memory size must be between %zu and %zu bytes.\n", + pool_overhead + block_size_min, pool_overhead + block_size_max); return 0; } From c13ef8561656291bfc0bd9edcb047eb76a67e922 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 30 Oct 2018 11:59:25 +0800 Subject: [PATCH 30/31] Increase heap size to 1TB to support larger sizes At the expense of more overhead in the TLSF structure. --- tlsf/tlsf.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c index cc1c555..3103982 100644 --- a/tlsf/tlsf.c +++ b/tlsf/tlsf.c @@ -62,11 +62,7 @@ enum tlsf_private { */ #if defined(TLSF_64BIT) - /* - * TODO: We can increase this to support larger sizes, at the expense - * of more overhead in the TLSF structure. - */ - FL_INDEX_MAX = 32, + FL_INDEX_MAX = 40, /* 1 TB */ #else FL_INDEX_MAX = 30, #endif From 2ce848ba4fc5abb8644eea867b7dd02fc9545b20 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 30 Oct 2018 12:11:17 +0800 Subject: [PATCH 31/31] Add API usage notice --- README.md | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c498ab8..38c9a01 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ Two Level Segregated Fit memory allocator implementation with O(1) time complexity, distributed under the BSD License. -Features --------- +## Features + * O(1) cost for malloc, free, realloc, memalign * Extremely low overhead per allocation (4 bytes) * Low overhead per TLSF management of pools (~3kB) @@ -12,20 +12,33 @@ Features * Compiles to only a few kB of code and data * Support for adding and removing memory pool regions on the fly -Caveats -------- +## API Usage + +You can essentially pass a memory pool for TLSF to manage. +```C +enum { STATIC_POOL_SIZE = 1 << 20 }; +static char s_pool[STATIC_POOL_SIZE]; + +tlsf_t instance = tlsf_create_with_pool(s_pool, STATIC_POOL_SIZE); + +void *data = tlsf_malloc(instance, 64); +tlsf_free(instance, data); +``` + +## Caveats + * Currently, assumes architecture can make 4-byte aligned accesses * Not designed to be thread safe; the user must provide this -Notes ------ +## Notes This code was based on the TLSF 1.4 spec and documentation found at: http://www.gii.upv.es/tlsf/main/docs -Reference: - M. Masmano, I. Ripoll, A. Crespo, and J. Real. - TLSF: a new dynamic memory allocator for real-time systems. - In Proc. ECRTS (2004), IEEE Computer Society, pp. 79-86. +## Reference + +M. Masmano, I. Ripoll, A. Crespo, and J. Real. +TLSF: a new dynamic memory allocator for real-time systems. +In Proc. ECRTS (2004), IEEE Computer Society, pp. 79-86. This implementation was written to the specification of the document, therefore no GPL restrictions apply. @@ -33,18 +46,18 @@ therefore no GPL restrictions apply. It also leverages the TLSF 2.0 improvement to shrink the per-block overhead from 8 to 4 bytes. -Known Issues ------------- +## Known Issues + * Due to the internal block structure size and the implementation -details of tlsf_memalign, there is worst-case behavior when requesting +details of `tlsf_memalign`, there is worst-case behavior when requesting small (<16 byte) blocks aligned to 8-byte boundaries. Overuse of memalign will generally increase fragmentation, but this particular case will leave lots of unusable "holes" in the pool. The solution would be to internally align all blocks to 8 bytes, but this will require significantl changes to the implementation. -Licensing -========= +## Licensing + TLSF-BSD is freely redistributable under the two-clause BSD License. Use of this source code is governed by a BSD-style license that can be found in the `LICENSE` file.