diff --git a/.clang-format b/.clang-format index 9705d27..2d07d20 100644 --- a/.clang-format +++ b/.clang-format @@ -1,13 +1,10 @@ BasedOnStyle: Chromium Language: Cpp MaxEmptyLinesToKeep: 3 -IndentCaseLabels: false AllowShortIfStatementsOnASingleLine: false -AllowShortCaseLabelsOnASingleLine: false AllowShortLoopsOnASingleLine: false DerivePointerAlignment: false PointerAlignment: Right -SpaceAfterCStyleCast: true TabWidth: 4 UseTab: Never IndentWidth: 4 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9decd57..0000000 --- a/.travis.yml +++ /dev/null @@ -1,71 +0,0 @@ -language: c - -dist: bionic - -script: make test - -matrix: - include: - - os: linux - addons: - apt: - sources: - - llvm-toolchain-trusty-3.9 - packages: - - clang-3.9 - env: - - MATRIX_EVAL="CC=clang-3.9" - - - os: linux - addons: - apt: - sources: - - llvm-toolchain-trusty-4.0 - packages: - - clang-4.0 - env: - - MATRIX_EVAL="CC=clang-4.0; export CFLAGS=-fsanitize=undefined" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-5 - env: - - MATRIX_EVAL="CC=gcc-5" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-6 - env: - - MATRIX_EVAL="CC=gcc-6" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-7 - env: - - MATRIX_EVAL="CC=gcc-7; export CFLAGS=-fsanitize=undefined" - - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-7-i686-linux-gnu - - gcc-7-multilib-i686-linux-gnu - env: - - MATRIX_EVAL="CC=i686-linux-gnu-gcc-7; export LDFLAGS=-static" - -before_install: - - eval "${MATRIX_EVAL}" diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..13fdd2b --- /dev/null +++ b/CHANGES @@ -0,0 +1,63 @@ +History +------- +2014/02/08 - v3.0 + * This version is based on improvements from 3DInteractive GmbH + * Interface changed to allow more than one memory pool + * Separated pool handling from control structure (adding, removing, debugging) + * Control structure and pools can still be constructed in the same memory block + * Memory blocks for control structure and pools are checked for alignment + * Added functions to retrieve control structure size, alignment size, min and + max block size, overhead of pool structure, and overhead of a single allocation + * Minimal Pool size is tlsf_block_size_min() + tlsf_pool_overhead() + * Pool must be empty when it is removed, in order to allow O(1) removal + +2011/10/20 - v2.0 + * 64-bit support + * More compiler intrinsics for ffs/fls + * ffs/fls verification during TLSF creation in debug builds + +2008/04/04 - v1.9 + * Add tlsf_heap_check, a heap integrity check + * Support a predefined tlsf_assert macro + * Fix realloc case where block should shrink; if adjacent block is + in use, execution would go down the slow path + +2007/02/08 - v1.8 + * Fix for unnecessary reallocation in tlsf_realloc + +2007/02/03 - v1.7 + * tlsf_heap_walk takes a callback + * tlsf_realloc now returns NULL on failure + * tlsf_memalign optimization for 4-byte alignment + * Usage of size_t where appropriate + +2006/11/21 - v1.6 + * ffs/fls broken out into tlsfbits.h + * tlsf_overhead queries per-pool overhead + +2006/11/07 - v1.5 + * Smart realloc implementation + * Smart memalign implementation + +2006/10/11 - v1.4 + * Add some ffs/fls implementations + * Minor code footprint reduction + +2006/09/14 - v1.3 + * Profiling indicates heavy use of blocks of + size 1-128, so implement small block handling + * Reduce pool overhead by about 1kb + * Reduce minimum block size from 32 to 12 bytes + * Realloc bug fix + +2006/09/09 - v1.2 + * Add tlsf_block_size + * Static assertion mechanism for invariants + * Minor bugfixes + +2006/09/01 - v1.1 + * Add tlsf_realloc + * Add tlsf_walk_heap + +2006/08/25 - v1.0 + * First release diff --git a/LICENSE b/LICENSE index 178db15..85fc745 100644 --- a/LICENSE +++ b/LICENSE @@ -1,30 +1,26 @@ -BSD 3-Clause License +TLSF-BSD is freely redistributable under the two-clause BSD License: -Copyright (c) 2006-2016, Matthew Conte -Copyright (c) 2017-2020, Daniel Mendler +Copyright (c) 2016 National Cheng Kung University, Taiwan. +Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. All rights reserved. Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. diff --git a/Makefile b/Makefile index e04fce5..2807a16 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,33 @@ OUT = build TARGETS = \ - test \ - bench + test-bits \ + test-alloc \ + bench-alloc TARGETS := $(addprefix $(OUT)/,$(TARGETS)) all: $(TARGETS) -test: all - ./build/bench - ./build/bench -s 32 - ./build/bench -s 10:12345 - ./build/test - -CFLAGS += \ - -std=gnu11 -g -O2 \ - -Wall -Wextra -Wshadow -Wpointer-arith -Wcast-qual -Wconversion -Wc++-compat \ - -DTLSF_ENABLE_ASSERT -DTLSF_ENABLE_CHECK +CC = gcc +CFLAGS = \ + -std=c99 -Wall -g -I tlsf \ + -D TLSF_CONFIG_ASSERT +LDFLAGS = OBJS = tlsf.o OBJS := $(addprefix $(OUT)/,$(OBJS)) deps := $(OBJS:%.o=%.o.d) -$(OUT)/test: $(OBJS) test.c +$(OUT)/test-%: $(OBJS) tests/test-%.c $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) +deps += build/test-bits.d -$(OUT)/bench: $(OBJS) bench.c +$(OUT)/bench-%: $(OBJS) tests/bench-%.c $(CC) $(CFLAGS) -o $@ -MMD -MF $@.d $^ $(LDFLAGS) +# FIXME: avoid hardcode +deps += build/bench-alloc.d -$(OUT)/%.o: %.c +$(OUT)/%.o: tlsf/%.c @mkdir -p $(OUT) $(CC) $(CFLAGS) -c -o $@ -MMD -MF $@.d $< @@ -39,6 +38,6 @@ check: $(TARGETS) clean: $(RM) $(TARGETS) $(OBJS) $(deps) -.PHONY: all check clean test +.PHONY: all check clean -include $(deps) diff --git a/README.md b/README.md index b13f9e6..38c9a01 100644 --- a/README.md +++ b/README.md @@ -1,71 +1,38 @@ -# tlsf-bsd: Two-Level Segregated Fit Memory Allocator +# TLSF-BSD -Two-Level Segregated Fit (TLSF) memory allocator implementation derived from the BSD-licensed implementation by [Matthew Conte](https://github.com/mattconte/tlsf). -This code was based on the [TLSF documentation](http://www.gii.upv.es/tlsf/main/docs.html). - -A novel technique called TLSF for dynamic memory allocation that maintains the effectiveness of the allocation and deallocation operations with a temporal cost of O(1). -For long-running applications, the fragmentation issue also has a greater influence on system performance. -The proposed method also produces a limited and tiny fragmentation. - -This implementation was written to the specification of the document, -therefore no GPL restrictions apply. +Two Level Segregated Fit memory allocator implementation with O(1) +time complexity, distributed under the BSD License. ## Features -* O(1) cost for `malloc`, `free`, `realloc`, `aligned_alloc` -* Low overhead per allocation (one word) -* Low overhead for the TLSF metadata (~4kB) + +* O(1) cost for malloc, free, realloc, memalign +* Extremely low overhead per allocation (4 bytes) +* Low overhead per TLSF management of pools (~3kB) * Low fragmentation -* Very small - only ~500 lines of code * Compiles to only a few kB of code and data -* Uses a linear memory area, which is resized on demand -* Not thread safe. API calls must be protected by a mutex in a multi-threaded environment. -* Works in environments with only minimal libc, uses only `stddef.h`, `stdbool.h`, `stdint.h` and `string.h`. - -## Design principals -1. Immediate coalescing: As soon as a block is freed, the algorithm is designed to merge the freed block with adjacent free blocks ,if any, to build up larger free block. -2. Splitting threshold: The smallest block of allocatable memory is 16 bytes. By this limit, it is possible to store the information needed to manage them, including the list of free blocks pointers. -3. Good-fit strategy: TLSF uses a large set of free lists where each list is a non-ordered list of free blocks whose size is between the size class and the new size class. Each segregated list contains blocks of the same class. -4. Same strategy for all block sizes: Some dynamic storage allocation (DSA) algorithms use different allocation strategies for different requested sizes. TLSF uses the same strategy for all sizes which provide uniform behavior, thus predictable WCET. -5. Memory is not cleaned-up: In multi-user environments, DSA algorithms have to clean the memory by usually filling with zeros in order to avoid security problems. In TLSF, since we assume the algorithm will be used in a trusted environment, the algorithm does not clean up the memory when allocating which would bring considerable overhead. - -## How it works +* Support for adding and removing memory pool regions on the fly -This package offers constant, O(1)-time memory block allocation and deallocation, by means of segregated fit mechanism. +## API Usage -Two-level structure to speedup access and reduce fragmentation. -* First level: Divides free blocks in classes of power of 2 -* Second level: Divides the size range indicated by the first level by 4. e.g., 2^6 first level covers the range of free list blocks of [2^6,2^7) - - This range is divided into 4 equidistant blocks. +You can essentially pass a memory pool for TLSF to manage. +```C +enum { STATIC_POOL_SIZE = 1 << 20 }; +static char s_pool[STATIC_POOL_SIZE]; -![TLSF Data Structure for Free Blocks](assets/data-structure.png) +tlsf_t instance = tlsf_create_with_pool(s_pool, STATIC_POOL_SIZE); -The structure consists of an array indexed by `log(2, requested_size)`. -In other words, requests are divided up according to the requsted size's most significant bit (MSB). -A pointer to the second level of the structure is contained in each item of the array. -At this level, the free blocks of each slab size are divided into x additional groups, -where x is a configurable number. -An array of size x that implements this partitioning is indexed by taking the value of the `log(2, x)` bits that follow the MSB. -Each value denotes the start of a linked list of free blocks (or is `NULL`). +void *data = tlsf_malloc(instance, 64); +tlsf_free(instance, data); +``` -Finding a free block in the correctly sized class (or, if none are available, in a larger size class) in constant time requires using the bitmaps representing the availability of free blocks (of a certain size class). +## Caveats -When `tlsf_free()` is called, the block examines if it may coalesce with nearby free blocks before returning to the free list. +* Currently, assumes architecture can make 4-byte aligned accesses +* Not designed to be thread safe; the user must provide this -### Finding a free block in TLSF `malloc()` - -TLSF searches for a free block for a request in the order: -1. First level index and second level index corresponding to the request is calculated. The indices are checked if a free block is available. If a free block is available at the indices, the block is returned. -2. If a free block is not available at the indices, remaining second level indices are searched to find a free block. If a free block is available, it is returned. -3. If not found, the next first level index whose value is 1 in the bitmap is searched to find a free block which guarantees to find a free block. - -#### Worst case happens when -1. The first level index calculated for the requested size is 1 and second level indices are examined which results in a fail to find a free block = or > the requested size. -2. The next free block available is on the right-most free-blocks list of the second level of the left-most first level index. When a small block is requested with size x, x bytes will be extracted from this huge block and returned. The remaining huge block going to the lower first level index results in the most overhead for this allocation operation. - -### Freeing a block in TLSF `free()` -1. When a block is freed, the first thing done is to check if the physical neighbour blocks are free or not. -2. If either of the neighbours are free, it is merged with the newly freed block. After the merge, new big block is inserted in the appropriate segregated list. (Mapping function is used to find first level and second level indices of the block) -3. If neither of the neighbours are free, only the freed block is put on to the appropriate place in the segregated list. +## Notes +This code was based on the TLSF 1.4 spec and documentation found at: + http://www.gii.upv.es/tlsf/main/docs ## Reference @@ -73,13 +40,24 @@ M. Masmano, I. Ripoll, A. Crespo, and J. Real. TLSF: a new dynamic memory allocator for real-time systems. In Proc. ECRTS (2004), IEEE Computer Society, pp. 79-86. -## Related Projects +This implementation was written to the specification of the document, +therefore no GPL restrictions apply. + +It also leverages the TLSF 2.0 improvement to shrink the per-block overhead +from 8 to 4 bytes. -* [tlsf-pmr](https://github.com/LiemDQ/tlsf-pmr): a memory resource for use with `polymorphic_allocator` that uses the Two-Level Segregated Fit algorithm as an allocation scheme. +## Known Issues +* Due to the internal block structure size and the implementation +details of `tlsf_memalign`, there is worst-case behavior when requesting +small (<16 byte) blocks aligned to 8-byte boundaries. Overuse of memalign +will generally increase fragmentation, but this particular case will leave +lots of unusable "holes" in the pool. The solution would be to internally +align all blocks to 8 bytes, but this will require significantl changes +to the implementation. ## Licensing -TLSF-BSD is freely redistributable under the 3-clause BSD License. +TLSF-BSD is freely redistributable under the two-clause BSD License. Use of this source code is governed by a BSD-style license that can be found in the `LICENSE` file. diff --git a/assets/data-structure.png b/assets/data-structure.png deleted file mode 100644 index 7c2ac0d..0000000 Binary files a/assets/data-structure.png and /dev/null differ diff --git a/bench.c b/bench.c deleted file mode 100644 index 1941008..0000000 --- a/bench.c +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2016 National Cheng Kung University, Taiwan. - * All rights reserved. - * Use of this source code is governed by a BSD-style license. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tlsf.h" - -static tlsf_t t = TLSF_INIT; - -static void usage(const char *name) -{ - printf( - "run a malloc benchmark.\n" - "usage: %s [-s blk-size|blk-min:blk-max] [-l loop-count] " - "[-n num-blocks] [-c]\n", - name); - exit(-1); -} - -/* Parse an integer argument. */ -static size_t parse_int_arg(const char *arg, const char *exe_name) -{ - long ret = strtol(arg, NULL, 0); - if (errno) - usage(exe_name); - - return (size_t) ret; -} - -/* Parse a size argument, which is either an integer or two integers separated - * by a colon, denoting a range. - */ -static void parse_size_arg(const char *arg, - const char *exe_name, - size_t *blk_min, - size_t *blk_max) -{ - char *endptr; - *blk_min = (size_t) strtol(arg, &endptr, 0); - - if (errno) - usage(exe_name); - - if (endptr && *endptr == ':') { - *blk_max = (size_t) strtol(endptr + 1, NULL, 0); - if (errno) - usage(exe_name); - } - - if (*blk_min > *blk_max) - usage(exe_name); -} - -/* Get a random block size between blk_min and blk_max. */ -static size_t get_random_block_size(size_t blk_min, size_t blk_max) -{ - if (blk_max > blk_min) - return blk_min + ((size_t) rand() % (blk_max - blk_min)); - return blk_min; -} - -static void run_alloc_benchmark(size_t loops, - size_t blk_min, - size_t blk_max, - void **blk_array, - size_t num_blks, - bool clear) -{ - while (loops--) { - size_t next_idx = (size_t) rand() % num_blks; - size_t blk_size = get_random_block_size(blk_min, blk_max); - - if (blk_array[next_idx]) { - if (rand() % 10 == 0) { - /* Insert the newly alloced block into the array at a random - * point. - */ - blk_array[next_idx] = - tlsf_realloc(&t, blk_array[next_idx], blk_size); - } else { - tlsf_free(&t, blk_array[next_idx]); - /* Insert the newly alloced block into the array at a random - * point. - */ - blk_array[next_idx] = tlsf_malloc(&t, blk_size); - } - } else { - /* Insert the newly alloced block into the array at a random point. - */ - blk_array[next_idx] = tlsf_malloc(&t, blk_size); - } - if (clear) - memset(blk_array[next_idx], 0, blk_size); - } - - /* Free up all allocated blocks. */ - for (size_t i = 0; i < num_blks; i++) { - if (blk_array[i]) - tlsf_free(&t, blk_array[i]); - } -} - -static size_t max_size; -static void *mem = 0; - -void *tlsf_resize(tlsf_t *_t, size_t req_size) -{ - (void) _t; - return req_size <= max_size ? mem : 0; -} - -int main(int argc, char **argv) -{ - size_t blk_min = 512, blk_max = 512, num_blks = 10000; - size_t loops = 10000000; - bool clear = false; - int opt; - - while ((opt = getopt(argc, argv, "s:l:r:t:n:b:ch")) > 0) { - switch (opt) { - case 's': - parse_size_arg(optarg, argv[0], &blk_min, &blk_max); - break; - case 'l': - loops = parse_int_arg(optarg, argv[0]); - break; - case 'n': - num_blks = parse_int_arg(optarg, argv[0]); - break; - case 'c': - clear = true; - break; - case 'h': - usage(argv[0]); - break; - default: - usage(argv[0]); - break; - } - } - - max_size = blk_max * num_blks; - mem = malloc(max_size); - - void **blk_array = (void **) calloc(num_blks, sizeof(void *)); - assert(blk_array); - - struct timespec start, end; - - int err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start); - assert(err == 0); - - printf("blk_min=%zu to blk_max=%zu\n", blk_min, blk_max); - - run_alloc_benchmark(loops, blk_min, blk_max, blk_array, num_blks, clear); - - err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end); - assert(err == 0); - free(blk_array); - - double elapsed = (double) (end.tv_sec - start.tv_sec) + - (double) (end.tv_nsec - start.tv_nsec) * 1e-9; - - struct rusage usage; - err = getrusage(RUSAGE_SELF, &usage); - assert(err == 0); - - /* Dump both machine and human readable versions */ - printf( - "%zu:%zu:%zu:%u:%lu:%.6f: took %.6f s for %zu malloc/free\nbenchmark " - "loops of %zu-%zu " - "bytes. ~%.3f us per loop\n", - blk_min, blk_max, loops, clear, usage.ru_maxrss, elapsed, elapsed, - loops, blk_min, blk_max, elapsed / (double) loops * 1e6); - - return 0; -} diff --git a/test.c b/test.c deleted file mode 100644 index ab2cb31..0000000 --- a/test.c +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2016 National Cheng Kung University, Taiwan. - * All rights reserved. - * Use of this source code is governed by a BSD-style license. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tlsf.h" - -static size_t PAGE; -static size_t MAX_PAGES; -static size_t curr_pages = 0; -static void *start_addr = 0; - -void *tlsf_resize(tlsf_t *t, size_t req_size) -{ - (void) t; - - if (!start_addr) - start_addr = mmap(0, MAX_PAGES * PAGE, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0); - - size_t req_pages = (req_size + PAGE - 1) / PAGE; - if (req_pages > MAX_PAGES) - return 0; - - if (req_pages != curr_pages) { - if (req_pages < curr_pages) - madvise((char *) start_addr + PAGE * req_pages, - (size_t) (curr_pages - req_pages) * PAGE, MADV_DONTNEED); - curr_pages = req_pages; - } - - return start_addr; -} - -static void random_test(tlsf_t *t, size_t spacelen, const size_t cap) -{ - const size_t maxitems = 2 * spacelen; - - void **p = (void **) malloc(maxitems * sizeof(void *)); - assert(p); - - /* Allocate random sizes up to the cap threshold. - * Track them in an array. - */ - int64_t rest = (int64_t) spacelen * (rand() % 6 + 1); - unsigned i = 0; - while (rest > 0) { - size_t len = ((size_t) rand() % cap) + 1; - if (rand() % 2 == 0) { - p[i] = tlsf_malloc(t, len); - } else { - size_t align = 1U << (rand() % 20); - if (cap < align) - align = 0; - else - len = align * (((size_t) rand() % (cap / align)) + 1); - p[i] = !align || !len ? tlsf_malloc(t, len) - : tlsf_aalloc(t, align, len); - if (align) - assert(!((size_t) p[i] % align)); - } - assert(p[i]); - rest -= (int64_t) len; - - if (rand() % 10 == 0) { - len = ((size_t) rand() % cap) + 1; - p[i] = tlsf_realloc(t, p[i], len); - assert(p[i]); - } - - tlsf_check(t); - - /* Fill with magic (only when testing up to 1MB). */ - uint8_t *data = (uint8_t *) p[i]; - if (spacelen <= 1024 * 1024) - memset(data, 0, len); - data[0] = 0xa5; - - if (i++ == maxitems) - break; - } - - /* Randomly deallocate the memory blocks until all of them are freed. - * The free space should match the free space after initialisation. - */ - for (unsigned n = i; n;) { - size_t target = (size_t) rand() % i; - if (p[target] == NULL) - continue; - uint8_t *data = (uint8_t *) p[target]; - assert(data[0] == 0xa5); - tlsf_free(t, p[target]); - p[target] = NULL; - n--; - - tlsf_check(t); - } - - free(p); -} - -#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0])) - -static void random_sizes_test(tlsf_t *t) -{ - const size_t sizes[] = {16, 32, 64, 128, 256, 512, 1024, 1024 * 1024}; - - for (unsigned i = 0; i < ARRAY_SIZE(sizes); i++) { - unsigned n = 1024; - - while (n--) { - size_t cap = (size_t) rand() % sizes[i] + 1; - printf("sizes = %zu, cap = %zu\n", sizes[i], cap); - random_test(t, sizes[i], cap); - } - } -} - -static void large_alloc(tlsf_t *t, size_t s) -{ - printf("large alloc %zu\n", s); - for (size_t d = 0; d < 100 && d < s; ++d) { - void *p = tlsf_malloc(t, s - d); - assert(p); - - void *q = tlsf_malloc(t, s - d); - assert(q); - tlsf_free(t, q); - - q = tlsf_malloc(t, s - d); - assert(q); - tlsf_free(t, q); - - tlsf_free(t, p); - tlsf_check(t); - } -} - -static void large_size_test(tlsf_t *t) -{ - size_t s = 1; - while (s <= TLSF_MAX_SIZE) { - large_alloc(t, s); - s *= 2; - } - - s = TLSF_MAX_SIZE; - while (s > 0) { - large_alloc(t, s); - s /= 2; - } -} - -static void append_pool_test(tlsf_t *t) -{ - printf("Pool append functionality test\n"); - - /* Simple test: Initial allocation */ - void *ptr1 = tlsf_malloc(t, 1000); - assert(ptr1); - - size_t initial_size = t->size; - - /* Try to append adjacent memory */ - void *append_addr = (char *) start_addr + initial_size; - size_t appended = tlsf_append_pool(t, append_addr, 4096); - - if (appended > 0) { - printf("Pool append successful: %zu bytes added\n", appended); - - /* Test large allocation from expanded pool */ - void *large_ptr = tlsf_malloc(t, 3000); - if (large_ptr) { - printf("Large allocation from expanded pool successful\n"); - tlsf_free(t, large_ptr); - } - } else { - printf("Pool append not possible (non-adjacent memory)\n"); - } - - /* Test non-adjacent append (should fail) */ - char separate_memory[2048]; - size_t non_adjacent = - tlsf_append_pool(t, separate_memory, sizeof(separate_memory)); - assert(non_adjacent == 0); - printf("Non-adjacent append correctly rejected\n"); - - tlsf_free(t, ptr1); - tlsf_check(t); - printf("Pool append test completed\n"); -} - -int main(void) -{ - PAGE = (size_t) sysconf(_SC_PAGESIZE); - MAX_PAGES = 20 * TLSF_MAX_SIZE / PAGE; - tlsf_t t = TLSF_INIT; - srand((unsigned int) time(0)); - - /* Run existing tests */ - large_size_test(&t); - random_sizes_test(&t); - - /* Run pool append test */ - append_pool_test(&t); - - puts("OK!"); - return 0; -} diff --git a/tests/bench-alloc.c b/tests/bench-alloc.c new file mode 100644 index 0000000..3d35ca5 --- /dev/null +++ b/tests/bench-alloc.c @@ -0,0 +1,227 @@ +/* Copyright (c) 2016 National Cheng Kung University, Taiwan. + * All rights reserved. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif +#define _POSIX_C_SOURCE 199309L + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lran2.h" +#include "tlsf.h" + +static tlsf_t *tlsf; + +static void usage(const char *name) +{ + printf("run a malloc benchmark.\n" + "usage: %s [-s blk-size|blk-min:blk-max] [-l loop-count] " + "[-n num-blocks] [-c]\n", + name); + exit(-1); +} + +/* Parse an integer argument. */ +static int parse_int_arg(const char *arg, const char *exe_name) +{ + long int ret; + + errno = 0; + ret = strtol(arg, NULL, 0); + if (errno) + usage(exe_name); + + return (int) ret; +} + +/* Parse a size argument, which is either an integer or two integers + separated by a colon, denoting a range. */ +static void +parse_size_arg(const char *arg, const char *exe_name, + size_t *blk_min, size_t *blk_max) +{ + long int ret; + char *endptr; + + errno = 0; + ret = strtol(arg, &endptr, 0); + + if (errno) + usage(exe_name); + + *blk_min = (int)ret; + + if (endptr && *endptr == ':') { + errno = 0; + ret = strtol(endptr + 1, NULL, 0); + + if (errno) + usage(exe_name); + } + + *blk_max = (int)ret; + + if (blk_min > blk_max) + usage(exe_name); +} + +/* Get a random block size between blk_min and blk_max. */ +static size_t +get_random_block_size(size_t blk_min, size_t blk_max, + struct lran2_st *lran2_state) +{ + size_t blk_size; + + if (blk_max > blk_min) { + blk_size = blk_min + (lran2(lran2_state) % (blk_max - blk_min)); + } else + blk_size = blk_min; + + return blk_size; +} + +static void +run_alloc_benchmark(int loops, size_t blk_min, size_t blk_max, + void **blk_array, size_t num_blks, bool clear, + struct lran2_st *lran2_state) +{ + while (loops--) { + int next_idx = lran2(lran2_state) % num_blks; + size_t blk_size = get_random_block_size(blk_min, blk_max, lran2_state); + + if (blk_array[next_idx]) + tlsf_free(tlsf, blk_array[next_idx]); + + /* Insert the newly alloced block into the array at a random point. */ + blk_array[next_idx] = tlsf_malloc(tlsf, blk_size); + if (clear) + memset(blk_array[next_idx], 0, blk_size); + } + + /* Free up all allocated blocks. */ + for (size_t i = 0; i < num_blks; i++) { + if (blk_array[i]) + tlsf_free(tlsf, blk_array[i]); + } +} + +struct alloc_desc { + /* Generic fields. */ + int loops; + size_t blk_min; + size_t blk_max; + void **blk_array; + size_t num_blks; + bool clear; +}; + +static void start_bench(void *arg) +{ + struct alloc_desc *desc = arg; + struct lran2_st lran2_state; + + lran2_init(&lran2_state, time(NULL) ^ getpid()); + + run_alloc_benchmark(desc->loops, desc->blk_min, desc->blk_max, + desc->blk_array, desc->num_blks, desc->clear, + &lran2_state); +} + +static void stop_bench(void *arg) +{ + struct alloc_desc *desc = arg; + if (!desc) return; + free(desc->blk_array); +} + +int main(int argc, char **argv) +{ + size_t blk_min = 512, blk_max = 512, num_blks = 10000; + int loops = 10000000; + bool clear = false; + int opt; + + while ((opt = getopt(argc, argv, "s:l:r:t:n:b:ch")) > 0) { + switch (opt) { + case 's': + parse_size_arg(optarg, argv[0], &blk_min, &blk_max); + break; + case 'l': + loops = parse_int_arg(optarg, argv[0]); + break; + case 'n': + num_blks = parse_int_arg(optarg, argv[0]); + break; + case 'c': + clear = true; + break; + case 'h': + usage(argv[0]); + break; + default: + usage(argv[0]); + break; + } + } + + const size_t overhead = tlsf_size() + tlsf_pool_overhead() + + tlsf_alloc_overhead(); + char *space = malloc(blk_max * num_blks + overhead); + assert(space); + tlsf = tlsf_create_with_pool(space, blk_max * num_blks + overhead); + + struct alloc_desc desc = { + .loops = loops, + .blk_min = blk_min, + .blk_max = blk_max, + .blk_array = malloc(num_blks * sizeof(unsigned char *)), + .num_blks = num_blks, + .clear = clear, + }; + assert(desc.blk_array != NULL); + memset(desc.blk_array, 0, num_blks * sizeof(unsigned char *)); + + struct timespec start, end; + + int err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &start); + assert(err == 0); + start_bench(&desc); + + err = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &end); + assert(err == 0); + stop_bench(&desc); + + double elapsed = (end.tv_sec - start.tv_sec) + + (end.tv_nsec - start.tv_nsec) * 1e-9; + + struct rusage usage; + err = getrusage(RUSAGE_SELF, &usage); + assert(err == 0); + + tlsf_destroy(tlsf); + free(space); + + /* Dump both machine and human readable versions */ + printf("%u:%u:%u:%u:%u:%.6f: took %.6f s for %u malloc/free\n" + "benchmark loops of %u-%u bytes. ~%.3f us per loop\n", + blk_min, blk_max, loops, + (int)clear, usage.ru_maxrss, elapsed, elapsed, loops, blk_min, + blk_max, (double)(elapsed / loops) * 1e6); + + return 0; +} diff --git a/tests/lran2.h b/tests/lran2.h new file mode 100644 index 0000000..34acdc8 --- /dev/null +++ b/tests/lran2.h @@ -0,0 +1,44 @@ +/* Copyright (c) 1996 Wolfram Gloger. + * A small, portable pseudo-random number generator. + */ + +#ifndef _LRAN2_H +#define _LRAN2_H + +#define LRAN2_MAX 714025l /* constants for portable */ +#define IA 1366l /* random number generator */ +#define IC 150889l /* (see e.g. `Numerical Recipes') */ + +struct lran2_st { + long x, y, v[97]; +}; + +static inline +void lran2_init(struct lran2_st *d, long seed) +{ + long x = (IC - seed) % LRAN2_MAX; + if (x < 0) + x = -x; + for (int j = 0; j < 97; j++) { + x = (IA * x + IC) % LRAN2_MAX; + d->v[j] = x; + } + d->x = (IA * x + IC) % LRAN2_MAX; + d->y = d->x; +} + +static inline +long lran2(struct lran2_st *d) +{ + int j = (d->y % 97); + + d->y = d->v[j]; + d->x = (IA * d->x + IC) % LRAN2_MAX; + d->v[j] = d->x; + return d->y; +} + +#undef IA +#undef IC + +#endif diff --git a/tests/test-alloc.c b/tests/test-alloc.c new file mode 100644 index 0000000..c0d704f --- /dev/null +++ b/tests/test-alloc.c @@ -0,0 +1,106 @@ +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static void random_test(const size_t spacelen, const size_t cap) +{ + const size_t overhead = tlsf_size() + tlsf_pool_overhead() + + tlsf_alloc_overhead(); + const size_t maxitems = spacelen; + size_t len; + uint8_t *space, *data; + unsigned i = 0; + tlsf_t *tlsf; + void **p; + + space = malloc(spacelen + overhead); + if (space == NULL) { + err(EXIT_FAILURE, "malloc"); + } + + p = malloc(maxitems * sizeof(void *)); + if (p == NULL) { + err(EXIT_FAILURE, "malloc"); + } + + tlsf = tlsf_create_with_pool(space, + spacelen + overhead); + assert(tlsf != NULL); + + /* + * Allocate random sizes up to the cap threshold. + * Track them in an array. + */ + for (;;) { + len = (random() % cap) + 1; + p[i] = tlsf_malloc(tlsf, len); + if (!p[i]) + break; + + /* Fill with magic (only when testing up to 1MB). */ + data = p[i]; + if (spacelen <= 1024 * 1024) { + memset(data, 0, len); + } + data[0] = 0xa5; + + if (i++ == maxitems) + break; + } + + /* + * Randomly deallocate the memory blocks until all of them are freed. + * The free space should match the free space after initialisation. + */ + for (unsigned n = i; n;) { + unsigned target = random() % i; + if (p[target] == NULL) + continue; + data = p[target]; + assert(data[0] == 0xa5); + tlsf_free(tlsf, p[target]); + p[target] = NULL; + n--; + } + + tlsf_destroy(tlsf); + free(space); + free(p); +} + +#define __arraycount(__x) \ + (sizeof(__x) / sizeof(__x[0])) + +static void random_sizes_test(void) +{ + const uint32_t sizes[] = {128, 1024, 1024 * 1024, 128 * 1024 * 1024}; + + for (unsigned i = 0; i < __arraycount(sizes); i++) { + unsigned n = 1024; + + while (n--) { + uint32_t cap = random() % sizes[i] + 1; + printf("sizes = %d, cap = %d\n", sizes[i], cap); + random_test(sizes[i], cap); + } + } +} + +int main(void) +{ + srandom(time(NULL) ^ getpid()); + random_sizes_test(); + puts("OK!"); + return 0; +} diff --git a/tests/test-bits.c b/tests/test-bits.c new file mode 100644 index 0000000..47dbbc0 --- /dev/null +++ b/tests/test-bits.c @@ -0,0 +1,31 @@ +#include +#include "tlsf_utils.h" + +int test_ffs_fls() +{ + /* Verify ffs/fls work properly. */ + int rv = 0; + rv += (tlsf_ffs(0) == -1) ? 0 : 0x1; + rv += (tlsf_fls(0) == -1) ? 0 : 0x2; + rv += (tlsf_ffs(1) == 0) ? 0 : 0x4; + rv += (tlsf_fls(1) == 0) ? 0 : 0x8; + rv += (tlsf_ffs(0x80000000) == 31) ? 0 : 0x10; + rv += (tlsf_ffs(0x80008000) == 15) ? 0 : 0x20; + rv += (tlsf_fls(0x80000008) == 31) ? 0 : 0x40; + rv += (tlsf_fls(0x7FFFFFFF) == 30) ? 0 : 0x80; + +#if defined(TLSF_64BIT) + rv += (tlsf_fls_sizet(0x80000000) == 31) ? 0 : 0x100; + rv += (tlsf_fls_sizet(0x100000000) == 32) ? 0 : 0x200; + rv += (tlsf_fls_sizet(0xffffffffffffffff) == 63) ? 0 : 0x400; +#endif + + if (rv) + printf("test_ffs_fls: %x ffs/fls tests failed.\n", rv); + return rv; +} + +int main() +{ + return test_ffs_fls(); +} diff --git a/tlsf.c b/tlsf.c deleted file mode 100644 index 1e37da7..0000000 --- a/tlsf.c +++ /dev/null @@ -1,756 +0,0 @@ -/* - * Copyright (c) 2006-2016, Matthew Conte - * Copyright (c) 2017-2020, Daniel Mendler - * All rights reserved. - * Use of this source code is governed by a BSD-style license. - */ - -#include -#include - -#include "tlsf.h" - -#ifndef UNLIKELY -#define UNLIKELY(x) __builtin_expect(!!(x), false) -#endif - -/* All allocation sizes and addresses are aligned. */ -#define ALIGN_SIZE ((size_t) 1 << ALIGN_SHIFT) -#if __SIZE_WIDTH__ == 64 -#define ALIGN_SHIFT 3 -#else -#define ALIGN_SHIFT 2 -#endif - -/* First level (FL) and second level (SL) counts */ -#define SL_SHIFT 4 -#define SL_COUNT (1U << SL_SHIFT) -#define FL_MAX _TLSF_FL_MAX -#define FL_SHIFT (SL_SHIFT + ALIGN_SHIFT) -#define FL_COUNT (FL_MAX - FL_SHIFT + 1) - -/* Block status bits are stored in the least significant bits (LSB) of the - * size field. - */ -#define BLOCK_BIT_FREE ((size_t) 1) -#define BLOCK_BIT_PREV_FREE ((size_t) 2) -#define BLOCK_BITS (BLOCK_BIT_FREE | BLOCK_BIT_PREV_FREE) - -/* A free block must be large enough to store its header minus the size of the - * prev field. - */ -#define BLOCK_OVERHEAD (sizeof(size_t)) -#define BLOCK_SIZE_MIN (sizeof(tlsf_block_t) - sizeof(tlsf_block_t *)) -#define BLOCK_SIZE_MAX ((size_t) 1 << (FL_MAX - 1)) -#define BLOCK_SIZE_SMALL ((size_t) 1 << FL_SHIFT) - -#ifndef ASSERT -#ifdef TLSF_ENABLE_ASSERT -#include -#define ASSERT(cond, msg) assert((cond) && msg) -#else -#define ASSERT(cond, msg) -#endif -#endif - -#ifndef INLINE -#define INLINE static inline __attribute__((always_inline)) -#endif - -typedef struct tlsf_block { - /* Points to the previous block. - * This field is only valid if the previous block is free and is actually - * stored at the end of the previous block. - */ - struct tlsf_block *prev; - - /* Size and block bits */ - size_t header; - - /* Next and previous free blocks. - * These fields are only valid if the corresponding block is free. - */ - struct tlsf_block *next_free, *prev_free; -} tlsf_block_t; - -_Static_assert(sizeof(size_t) == 4 || sizeof(size_t) == 8, - "size_t must be 32 or 64 bit"); -_Static_assert(sizeof(size_t) == sizeof(void *), - "size_t must equal pointer size"); -_Static_assert(ALIGN_SIZE == BLOCK_SIZE_SMALL / SL_COUNT, - "sizes are not properly set"); -_Static_assert(BLOCK_SIZE_MIN < BLOCK_SIZE_SMALL, - "min allocation size is wrong"); -_Static_assert(BLOCK_SIZE_MAX == TLSF_MAX_SIZE + BLOCK_OVERHEAD, - "max allocation size is wrong"); -_Static_assert(FL_COUNT <= 32, "index too large"); -_Static_assert(SL_COUNT <= 32, "index too large"); -_Static_assert(FL_COUNT == _TLSF_FL_COUNT, "invalid level configuration"); -_Static_assert(SL_COUNT == _TLSF_SL_COUNT, "invalid level configuration"); - -INLINE uint32_t bitmap_ffs(uint32_t x) -{ - uint32_t i = (uint32_t) __builtin_ffs((int32_t) x); - ASSERT(i, "no set bit found"); - return i - 1U; -} - -INLINE uint32_t log2floor(size_t x) -{ - ASSERT(x > 0, "log2 of zero"); -#if __SIZE_WIDTH__ == 64 - return (uint32_t) (63 - (uint32_t) __builtin_clzll((unsigned long long) x)); -#else - return (uint32_t) (31 - (uint32_t) __builtin_clzl((unsigned long) x)); -#endif -} - -INLINE size_t block_size(const tlsf_block_t *block) -{ - return block->header & ~BLOCK_BITS; -} - -INLINE void block_set_size(tlsf_block_t *block, size_t size) -{ - ASSERT(!(size % ALIGN_SIZE), "invalid size"); - block->header = size | (block->header & BLOCK_BITS); -} - -INLINE bool block_is_free(const tlsf_block_t *block) -{ - return !!(block->header & BLOCK_BIT_FREE); -} - -INLINE bool block_is_prev_free(const tlsf_block_t *block) -{ - return !!(block->header & BLOCK_BIT_PREV_FREE); -} - -INLINE void block_set_prev_free(tlsf_block_t *block, bool free) -{ - block->header = free ? block->header | BLOCK_BIT_PREV_FREE - : block->header & ~BLOCK_BIT_PREV_FREE; -} - -INLINE size_t align_up(size_t x, size_t align) -{ - ASSERT(!(align & (align - 1)), "must align to a power of two"); - return (((x - 1) | (align - 1)) + 1); -} - -INLINE char *align_ptr(char *p, size_t align) -{ - return (char *) align_up((size_t) p, align); -} - -INLINE char *block_payload(tlsf_block_t *block) -{ - return (char *) block + offsetof(tlsf_block_t, header) + BLOCK_OVERHEAD; -} - -INLINE tlsf_block_t *to_block(void *ptr) -{ - tlsf_block_t *block = (tlsf_block_t *) ptr; - ASSERT(block_payload(block) == align_ptr(block_payload(block), ALIGN_SIZE), - "block not aligned properly"); - return block; -} - -INLINE tlsf_block_t *block_from_payload(void *ptr) -{ - return to_block((char *) ptr - offsetof(tlsf_block_t, header) - - BLOCK_OVERHEAD); -} - -/* Return location of previous block. */ -INLINE tlsf_block_t *block_prev(const tlsf_block_t *block) -{ - ASSERT(block_is_prev_free(block), "previous block must be free"); - return block->prev; -} - -/* Return location of next existing block. */ -INLINE tlsf_block_t *block_next(tlsf_block_t *block) -{ - tlsf_block_t *next = - to_block(block_payload(block) + block_size(block) - BLOCK_OVERHEAD); - ASSERT(block_size(block), "block is last"); - return next; -} - -/* Link a new block with its neighbor, return the neighbor. */ -INLINE tlsf_block_t *block_link_next(tlsf_block_t *block) -{ - tlsf_block_t *next = block_next(block); - next->prev = block; - return next; -} - -INLINE bool block_can_split(tlsf_block_t *block, size_t size) -{ - return block_size(block) >= sizeof(tlsf_block_t) + size; -} - -INLINE void block_set_free(tlsf_block_t *block, bool free) -{ - ASSERT(block_is_free(block) != free, "block free bit unchanged"); - block->header = - free ? block->header | BLOCK_BIT_FREE : block->header & ~BLOCK_BIT_FREE; - block_set_prev_free(block_link_next(block), free); -} - -/* Adjust allocation size to be aligned, and no smaller than internal minimum. - */ -INLINE size_t adjust_size(size_t size, size_t align) -{ - size = align_up(size, align); - return size < BLOCK_SIZE_MIN ? BLOCK_SIZE_MIN : size; -} - -/* Round up to the next block size */ -INLINE size_t round_block_size(size_t size) -{ - size_t t = ((size_t) 1 << (log2floor(size) - SL_SHIFT)) - 1; - return size >= BLOCK_SIZE_SMALL ? (size + t) & ~t : size; -} - -INLINE void mapping(size_t size, uint32_t *fl, uint32_t *sl) -{ - if (size < BLOCK_SIZE_SMALL) { - /* Store small blocks in first list. */ - *fl = 0; - *sl = (uint32_t) size / (BLOCK_SIZE_SMALL / SL_COUNT); - } else { - uint32_t t = log2floor(size); - *sl = (uint32_t) (size >> (t - SL_SHIFT)) ^ SL_COUNT; - *fl = t - FL_SHIFT + 1; - } - ASSERT(*fl < FL_COUNT, "wrong first level"); - ASSERT(*sl < SL_COUNT, "wrong second level"); -} - -/* Calculate the minimum block size for a given FL/SL bin */ -INLINE size_t mapping_size(uint32_t fl, uint32_t sl) -{ - if (fl == 0) - return sl * (BLOCK_SIZE_SMALL / SL_COUNT); - - size_t size = (size_t) 1 << (fl + FL_SHIFT - 1); - return size + (sl * (size >> SL_SHIFT)); -} - -INLINE tlsf_block_t *block_find_suitable(tlsf_t *t, uint32_t *fl, uint32_t *sl) -{ - ASSERT(*fl < FL_COUNT, "wrong first level"); - ASSERT(*sl < SL_COUNT, "wrong second level"); - - /* Search for a block in the list associated with the given fl/sl index. */ - uint32_t sl_map = t->sl[*fl] & (~0U << *sl); - if (!sl_map) { - /* No block exists. Search in the next largest first-level list. */ - uint32_t fl_map = t->fl & ((*fl + 1 >= 32) ? 0U : (~0U << (*fl + 1))); - - /* No free blocks available, memory has been exhausted. */ - if (UNLIKELY(!fl_map)) - return NULL; - - *fl = bitmap_ffs(fl_map); - ASSERT(*fl < FL_COUNT, "wrong first level"); - - sl_map = t->sl[*fl]; - ASSERT(sl_map, "second level bitmap is null"); - } - - *sl = bitmap_ffs(sl_map); - ASSERT(*sl < SL_COUNT, "wrong second level"); - - return t->block[*fl][*sl]; -} - -/* Remove a free block from the free list. */ -INLINE void remove_free_block(tlsf_t *t, - tlsf_block_t *block, - uint32_t fl, - uint32_t sl) -{ - ASSERT(fl < FL_COUNT, "wrong first level"); - ASSERT(sl < SL_COUNT, "wrong second level"); - - tlsf_block_t *prev = block->prev_free; - tlsf_block_t *next = block->next_free; - if (next) - next->prev_free = prev; - if (prev) - prev->next_free = next; - - /* If this block is the head of the free list, set new head. */ - if (t->block[fl][sl] == block) { - t->block[fl][sl] = next; - - /* If the new head is null, clear the bitmap. */ - if (!next) { - t->sl[fl] &= ~(1U << sl); - - /* If the second bitmap is now empty, clear the fl bitmap. */ - if (!t->sl[fl]) - t->fl &= ~(1U << fl); - } - } -} - -/* Insert a free block into the free block list and mark the bitmaps. */ -INLINE void insert_free_block(tlsf_t *t, - tlsf_block_t *block, - uint32_t fl, - uint32_t sl) -{ - tlsf_block_t *current = t->block[fl][sl]; - ASSERT(block, "cannot insert a null entry into the free list"); - block->next_free = current; - block->prev_free = 0; - if (current) - current->prev_free = block; - t->block[fl][sl] = block; - t->fl |= 1U << fl; - t->sl[fl] |= 1U << sl; -} - -/* Remove a given block from the free list. */ -INLINE void block_remove(tlsf_t *t, tlsf_block_t *block) -{ - uint32_t fl, sl; - mapping(block_size(block), &fl, &sl); - remove_free_block(t, block, fl, sl); -} - -/* Insert a given block into the free list. */ -INLINE void block_insert(tlsf_t *t, tlsf_block_t *block) -{ - uint32_t fl, sl; - mapping(block_size(block), &fl, &sl); - insert_free_block(t, block, fl, sl); -} - -/* Split a block into two, the second of which is free. */ -INLINE tlsf_block_t *block_split(tlsf_block_t *block, size_t size) -{ - tlsf_block_t *rest = to_block(block_payload(block) + size - BLOCK_OVERHEAD); - size_t rest_size = block_size(block) - (size + BLOCK_OVERHEAD); - ASSERT(block_size(block) == rest_size + size + BLOCK_OVERHEAD, - "rest block size is wrong"); - ASSERT(rest_size >= BLOCK_SIZE_MIN, "block split with invalid size"); - rest->header = rest_size; - ASSERT(!(rest_size % ALIGN_SIZE), "invalid block size"); - block_set_free(rest, true); - block_set_size(block, size); - return rest; -} - -/* Absorb a free block's storage into an adjacent previous free block. */ -INLINE tlsf_block_t *block_absorb(tlsf_block_t *prev, tlsf_block_t *block) -{ - ASSERT(block_size(prev), "previous block can't be last"); - /* Note: Leaves flags untouched. */ - prev->header += block_size(block) + BLOCK_OVERHEAD; - block_link_next(prev); - return prev; -} - -/* Merge a just-freed block with an adjacent previous free block. */ -INLINE tlsf_block_t *block_merge_prev(tlsf_t *t, tlsf_block_t *block) -{ - if (block_is_prev_free(block)) { - tlsf_block_t *prev = block_prev(block); - ASSERT(prev, "prev block can't be null"); - ASSERT(block_is_free(prev), - "prev block is not free though marked as such"); - block_remove(t, prev); - block = block_absorb(prev, block); - } - return block; -} - -/* Merge a just-freed block with an adjacent free block. */ -INLINE tlsf_block_t *block_merge_next(tlsf_t *t, tlsf_block_t *block) -{ - tlsf_block_t *next = block_next(block); - ASSERT(next, "next block can't be null"); - if (block_is_free(next)) { - ASSERT(block_size(block), "previous block can't be last"); - block_remove(t, next); - block = block_absorb(block, next); - } - return block; -} - -/* Trim any trailing block space off the end of a block, return to pool. */ -INLINE void block_rtrim_free(tlsf_t *t, tlsf_block_t *block, size_t size) -{ - ASSERT(block_is_free(block), "block must be free"); - if (!block_can_split(block, size)) - return; - tlsf_block_t *rest = block_split(block, size); - block_link_next(block); - block_set_prev_free(rest, true); - block_insert(t, rest); -} - -/* Trim any trailing block space off the end of a used block, return to pool. */ -INLINE void block_rtrim_used(tlsf_t *t, tlsf_block_t *block, size_t size) -{ - ASSERT(!block_is_free(block), "block must be used"); - if (!block_can_split(block, size)) - return; - tlsf_block_t *rest = block_split(block, size); - block_set_prev_free(rest, false); - rest = block_merge_next(t, rest); - block_insert(t, rest); -} - -INLINE tlsf_block_t *block_ltrim_free(tlsf_t *t, - tlsf_block_t *block, - size_t size) -{ - ASSERT(block_is_free(block), "block must be free"); - ASSERT(block_can_split(block, size), "block is too small"); - tlsf_block_t *rest = block_split(block, size - BLOCK_OVERHEAD); - block_set_prev_free(rest, true); - block_link_next(block); - block_insert(t, block); - return rest; -} - -INLINE void *block_use(tlsf_t *t, tlsf_block_t *block, size_t size) -{ - block_rtrim_free(t, block, size); - block_set_free(block, false); - return block_payload(block); -} - -INLINE void check_sentinel(tlsf_block_t *block) -{ - (void) block; - ASSERT(!block_size(block), "sentinel should be last"); - ASSERT(!block_is_free(block), "sentinel block should not be free"); -} - -static bool arena_grow(tlsf_t *t, size_t size) -{ - size_t req_size = - (t->size ? t->size + BLOCK_OVERHEAD : 2 * BLOCK_OVERHEAD) + size; - void *addr = tlsf_resize(t, req_size); - if (!addr) - return false; - ASSERT((size_t) addr % ALIGN_SIZE == 0, "wrong heap alignment address"); - tlsf_block_t *block = - to_block(t->size ? (char *) addr + t->size - 2 * BLOCK_OVERHEAD - : (char *) addr - BLOCK_OVERHEAD); - if (!t->size) - block->header = 0; - check_sentinel(block); - block->header |= size | BLOCK_BIT_FREE; - block = block_merge_prev(t, block); - block_insert(t, block); - tlsf_block_t *sentinel = block_link_next(block); - sentinel->header = BLOCK_BIT_PREV_FREE; - t->size = req_size; - check_sentinel(sentinel); - return true; -} - -static size_t arena_append_pool(tlsf_t *t, void *mem, size_t size) -{ - if (!t->size || !mem || size < 2 * BLOCK_OVERHEAD) - return 0; - - /* Align memory block boundaries */ - char *start = align_ptr((char *) mem, ALIGN_SIZE); - char *end = (char *) mem + size; - size_t aligned_size = (size_t) (end - start) & ~(ALIGN_SIZE - 1); - - if (aligned_size < 2 * BLOCK_OVERHEAD) - return 0; - - /* Get current pool information */ - void *current_pool_start = - tlsf_resize(t, t->size); /* Ensure current pool is available */ - if (!current_pool_start) - return 0; - - char *current_pool_end = (char *) current_pool_start + t->size; - - /* Only support coalescing if the new memory is immediately adjacent to the - * current pool - */ - if (start != current_pool_end) - return 0; - - /* Update the pool size first to include the new memory */ - size_t old_size = t->size; - size_t new_total_size = t->size + aligned_size; - - /* Try to resize the pool to include the new memory */ - void *resized_pool = tlsf_resize(t, new_total_size); - if (!resized_pool) - return 0; - - /* Update our pool size */ - t->size = new_total_size; - - /* Find the current sentinel block */ - tlsf_block_t *old_sentinel = - to_block((char *) resized_pool + old_size - 2 * BLOCK_OVERHEAD); - check_sentinel(old_sentinel); - - /* Check if the block before the sentinel is free */ - tlsf_block_t *last_block = NULL; - if (block_is_prev_free(old_sentinel)) { - last_block = block_prev(old_sentinel); - ASSERT(last_block && block_is_free(last_block), - "last block should be free"); - /* Remove the last free block from lists since we'll recreate it */ - block_remove(t, last_block); - } - - /* Calculate the new free block size */ - size_t new_free_size = - aligned_size + BLOCK_OVERHEAD; /* Include old sentinel space */ - tlsf_block_t *new_free_block; - - if (last_block) { - /* Merge with the existing free block */ - new_free_size += block_size(last_block) + BLOCK_OVERHEAD; - new_free_block = last_block; - } else { - /* Convert the old sentinel into the start of the new free block */ - new_free_block = old_sentinel; - } - - /* Set up the new free block header */ - new_free_block->header = new_free_size | BLOCK_BIT_FREE; - - /* Set up proper linking for the new free block */ - if (!last_block && old_size > 2 * BLOCK_OVERHEAD) { - /* There's a previous block, find it by scanning backwards */ - char *scan_start = (char *) old_sentinel - BLOCK_OVERHEAD; - char *pool_start = (char *) resized_pool; - - /* Simple backward scan to find the previous block */ - for (char *scan_ptr = scan_start; scan_ptr >= pool_start; - scan_ptr -= ALIGN_SIZE) { - tlsf_block_t *candidate = to_block(scan_ptr - BLOCK_OVERHEAD); - if ((char *) candidate >= pool_start && - (char *) candidate + BLOCK_OVERHEAD + block_size(candidate) == - (char *) old_sentinel) { - new_free_block->prev = candidate; - block_set_prev_free(new_free_block, block_is_free(candidate)); - break; - } - } - } - - /* Insert the new free block into the appropriate list */ - block_insert(t, new_free_block); - - /* Create a new sentinel at the end */ - tlsf_block_t *new_sentinel = block_link_next(new_free_block); - new_sentinel->header = BLOCK_BIT_PREV_FREE; - check_sentinel(new_sentinel); - - return aligned_size; -} - -static void arena_shrink(tlsf_t *t, tlsf_block_t *block) -{ - check_sentinel(block_next(block)); - size_t size = block_size(block); - ASSERT(t->size + BLOCK_OVERHEAD >= size, "invalid heap size before shrink"); - t->size = t->size - size - BLOCK_OVERHEAD; - if (t->size == BLOCK_OVERHEAD) - t->size = 0; - tlsf_resize(t, t->size); - if (t->size) { - block->header = 0; - check_sentinel(block); - } -} - -INLINE tlsf_block_t *block_find_free(tlsf_t *t, size_t *size) -{ - *size = round_block_size(*size); - uint32_t fl, sl; - mapping(*size, &fl, &sl); - tlsf_block_t *block = block_find_suitable(t, &fl, &sl); - if (UNLIKELY(!block)) { - if (!arena_grow(t, *size)) - return NULL; - block = block_find_suitable(t, &fl, &sl); - ASSERT(block, "no block found"); - } - - /* Update size to match the FL/SL bin that was actually used. - * This ensures that when the block is freed, it will be placed in the same - * bin it was allocated from. - */ - *size = mapping_size(fl, sl); - ASSERT(block_size(block) >= *size, "insufficient block size"); - remove_free_block(t, block, fl, sl); - return block; -} - -void *tlsf_malloc(tlsf_t *t, size_t size) -{ - size = adjust_size(size, ALIGN_SIZE); - if (UNLIKELY(size > TLSF_MAX_SIZE)) - return NULL; - tlsf_block_t *block = block_find_free(t, &size); - if (UNLIKELY(!block)) - return NULL; - return block_use(t, block, size); -} - -void *tlsf_aalloc(tlsf_t *t, size_t align, size_t size) -{ - size_t adjust = adjust_size(size, ALIGN_SIZE); - - if (UNLIKELY( - !size || - ((align | size) & (align - 1)) /* align!=2**x, size!=n*align */ || - align > TLSF_MAX_SIZE || sizeof(tlsf_block_t) > TLSF_MAX_SIZE || - adjust > TLSF_MAX_SIZE - align - - sizeof(tlsf_block_t) /* size is too large */)) - return NULL; - - if (align <= ALIGN_SIZE) - return tlsf_malloc(t, size); - - size_t asize = - adjust_size(adjust + align - 1 + sizeof(tlsf_block_t), align); - tlsf_block_t *block = block_find_free(t, &asize); - if (UNLIKELY(!block)) - return NULL; - - char *mem = align_ptr(block_payload(block) + sizeof(tlsf_block_t), align); - block = block_ltrim_free(t, block, (size_t) (mem - block_payload(block))); - return block_use(t, block, adjust); -} - -void tlsf_free(tlsf_t *t, void *mem) -{ - if (UNLIKELY(!mem)) - return; - - tlsf_block_t *block = block_from_payload(mem); - ASSERT(!block_is_free(block), "block already marked as free"); - - block_set_free(block, true); - block = block_merge_prev(t, block); - block = block_merge_next(t, block); - - if (UNLIKELY(!block_size(block_next(block)))) - arena_shrink(t, block); - else - block_insert(t, block); -} - -void *tlsf_realloc(tlsf_t *t, void *mem, size_t size) -{ - /* Zero-size requests are treated as free. */ - if (UNLIKELY(mem && !size)) { - tlsf_free(t, mem); - return NULL; - } - - /* Null-pointer requests are treated as malloc. */ - if (UNLIKELY(!mem)) - return tlsf_malloc(t, size); - - tlsf_block_t *block = block_from_payload(mem); - size_t avail = block_size(block); - size = adjust_size(size, ALIGN_SIZE); - if (UNLIKELY(size > TLSF_MAX_SIZE)) - return NULL; - - ASSERT(!block_is_free(block), "block already marked as free"); - - /* Do we need to expand to the next block? */ - if (size > avail) { - /* If the next block is used or too small, we must relocate and copy. */ - tlsf_block_t *next = block_next(block); - if (!block_is_free(next) || - size > avail + block_size(next) + BLOCK_OVERHEAD) { - void *dst = tlsf_malloc(t, size); - if (dst) { - memcpy(dst, mem, avail); - tlsf_free(t, mem); - } - return dst; - } - - block_merge_next(t, block); - block_set_prev_free(block_next(block), false); - } - - /* Trim the resulting block and return the original pointer. */ - block_rtrim_used(t, block, size); - return mem; -} - -size_t tlsf_append_pool(tlsf_t *t, void *mem, size_t size) -{ - if (UNLIKELY(!t || !mem || !size)) - return 0; - - return arena_append_pool(t, mem, size); -} - -#ifdef TLSF_ENABLE_CHECK -#include -#include -#define CHECK(cond, msg) \ - ({ \ - if (!(cond)) { \ - fprintf(stderr, "TLSF CHECK: %s - %s\n", msg, #cond); \ - abort(); \ - } \ - }) -void tlsf_check(tlsf_t *t) -{ - for (uint32_t i = 0; i < FL_COUNT; ++i) { - for (uint32_t j = 0; j < SL_COUNT; ++j) { - size_t fl_map = t->fl & (1U << i), sl_list = t->sl[i], - sl_map = sl_list & (1U << j); - tlsf_block_t *block = t->block[i][j]; - - /* Check that first- and second-level lists agree. */ - if (!fl_map) - CHECK(!sl_map, "second-level map must be null"); - - if (!sl_map) { - CHECK(!block, "block list must be null"); - continue; - } - - /* Check that there is at least one free block. */ - CHECK(sl_list, "no free blocks in second-level map"); - - while (block) { - uint32_t fl, sl; - CHECK(block_is_free(block), "block should be free"); - CHECK(!block_is_prev_free(block), - "blocks should have coalesced"); - CHECK(!block_is_free(block_next(block)), - "blocks should have coalesced"); - CHECK(block_is_prev_free(block_next(block)), - "block should be free"); - CHECK(block_size(block) >= BLOCK_SIZE_MIN, - "block not minimum size"); - - mapping(block_size(block), &fl, &sl); - CHECK(fl == i && sl == j, "block size indexed in wrong list"); - block = block->next_free; - } - } - } -} -#endif diff --git a/tlsf.h b/tlsf.h deleted file mode 100644 index 40de132..0000000 --- a/tlsf.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * SPDX-License-Identifier: BSD-3-Clause - */ - -#pragma once - -/* Inhibit C++ name-mangling for tlsf functions */ -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#include -#include - -#define _TLSF_SL_COUNT 16 -#if __SIZE_WIDTH__ == 64 -#define _TLSF_FL_COUNT 32 -#define _TLSF_FL_MAX 38 -#else -#define _TLSF_FL_COUNT 25 -#define _TLSF_FL_MAX 30 -#endif -#define TLSF_MAX_SIZE (((size_t) 1 << (_TLSF_FL_MAX - 1)) - sizeof(size_t)) -#define TLSF_INIT ((tlsf_t) {.size = 0}) - -typedef struct { - uint32_t fl, sl[_TLSF_FL_COUNT]; - struct tlsf_block *block[_TLSF_FL_COUNT][_TLSF_SL_COUNT]; - size_t size; -} tlsf_t; - -void *tlsf_resize(tlsf_t *, size_t); -void *tlsf_aalloc(tlsf_t *, size_t, size_t); - -/** - * Append a memory block to an existing pool, potentially coalescing with - * the last block if it's free. Returns the number of bytes actually used - * from the memory block for pool expansion. - * - * @param tlsf The TLSF allocator instance - * @param mem Pointer to the memory block to append - * @param size Size of the memory block in bytes - * @return Number of bytes used from the memory block, 0 on failure - */ -size_t tlsf_append_pool(tlsf_t *tlsf, void *mem, size_t size); - -/** - * Allocates the requested @size bytes of memory and returns a pointer to it. - * On failure, returns NULL. - */ -void *tlsf_malloc(tlsf_t *, size_t size); -void *tlsf_realloc(tlsf_t *, void *, size_t); - -/** - * Releases the previously allocated memory, given the pointer. - */ -void tlsf_free(tlsf_t *, void *); - -#ifdef TLSF_ENABLE_CHECK -void tlsf_check(tlsf_t *); -#else -static inline void tlsf_check(tlsf_t *t) -{ - (void) t; -} -#endif - -#ifdef __cplusplus -} -#endif diff --git a/tlsf/tlsf.c b/tlsf/tlsf.c new file mode 100644 index 0000000..3103982 --- /dev/null +++ b/tlsf/tlsf.c @@ -0,0 +1,1045 @@ +/* Copyright (c) 2016 National Cheng Kung University, Taiwan. + * Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. + * All rights reserved. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include + +#include "tlsf.h" + +#include "tlsf_utils.h" + +#if __GNUC__ || __INTEL_COMPILER +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) (x) +#define unlikely(x) (x) +#endif + +#ifdef TLSF_CONFIG_ASSERT +#include +#define tlsf_assert(expr) assert(expr) +#else +#define tlsf_assert(expr) (void)(0) +#endif + +/* Public constants: may be modified. */ +enum tlsf_public { + /* log2 of number of linear subdivisions of block sizes. Larger + * values require more memory in the control structure. Values of + * 4 or 5 are typical. + */ + SL_INDEX_COUNT_LOG2 = 5, +}; + +/* Private constants: do not modify. */ +enum tlsf_private { +#if defined(TLSF_64BIT) + /* All allocation sizes and addresses are aligned to 8 bytes. */ + ALIGN_SIZE_LOG2 = 3, +#else + /* All allocation sizes and addresses are aligned to 4 bytes. */ + ALIGN_SIZE_LOG2 = 2, +#endif + ALIGN_SIZE = (1 << ALIGN_SIZE_LOG2), + +/* + * We support allocations of sizes up to (1 << FL_INDEX_MAX) bits. + * However, because we linearly subdivide the second-level lists, and + * our minimum size granularity is 4 bytes, it doesn't make sense to + * create first-level lists for sizes smaller than SL_INDEX_COUNT * 4, + * or (1 << (SL_INDEX_COUNT_LOG2 + 2)) bytes, as there we will be + * trying to split size ranges into more slots than we have available. + * Instead, we calculate the minimum threshold size, and place all + * blocks below that size into the 0th first-level list. + */ + +#if defined(TLSF_64BIT) + FL_INDEX_MAX = 40, /* 1 TB */ +#else + FL_INDEX_MAX = 30, +#endif + SL_INDEX_COUNT = (1 << SL_INDEX_COUNT_LOG2), + FL_INDEX_SHIFT = (SL_INDEX_COUNT_LOG2 + ALIGN_SIZE_LOG2), + FL_INDEX_COUNT = (FL_INDEX_MAX - FL_INDEX_SHIFT + 1), + + SMALL_BLOCK_SIZE = (1 << FL_INDEX_SHIFT), +}; + +/* + * Cast and min/max macros. + */ +#define tlsf_cast(t, exp) ((t)(exp)) +#define tlsf_min(a, b) ((a) < (b) ? (a) : (b)) +#define tlsf_max(a, b) ((a) > (b) ? (a) : (b)) + +/* + * Static assertion mechanism. + */ +#define _tlsf_glue2(x, y) x##y +#define _tlsf_glue(x, y) _tlsf_glue2(x, y) +#define tlsf_static_assert(exp) \ + typedef char _tlsf_glue(static_assert, __LINE__)[(exp) ? 1 : -1] + +/* This code has been tested on 32- and 64-bit (LP/LLP) architectures. */ +tlsf_static_assert(sizeof(int) * CHAR_BIT == 32); +tlsf_static_assert(sizeof(size_t) * CHAR_BIT >= 32); +tlsf_static_assert(sizeof(size_t) * CHAR_BIT <= 64); + +/* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */ +tlsf_static_assert(sizeof(unsigned int) * CHAR_BIT >= SL_INDEX_COUNT); + +/* Ensure we've properly tuned our sizes. */ +tlsf_static_assert(ALIGN_SIZE == SMALL_BLOCK_SIZE / SL_INDEX_COUNT); + +/* + * Data structures and associated constants. + */ + +/* + * Block header structure. + * + * There are several implementation subtleties involved: + * - The prev_phys_block field is only valid if the previous block is free. + * - The prev_phys_block field is actually stored at the end of the + * previous block. It appears at the beginning of this structure only to + * simplify the implementation. + * - The next_free / prev_free fields are only valid if the block is free. + */ +typedef struct block_header_t { + /* Points to the previous physical block. */ + struct block_header_t *prev_phys_block; + + /* The size of this block, excluding the block header. */ + size_t size; + + /* Next and previous free blocks. */ + struct block_header_t *next_free; + struct block_header_t *prev_free; +} block_header_t; + +/* + * Since block sizes are always at least a multiple of 4, the two least + * significant bits of the size field are used to store the block status: + * - bit 0: whether block is busy or free + * - bit 1: whether previous block is busy or free + */ +static const size_t block_header_free_bit = 1 << 0; +static const size_t block_header_prev_free_bit = 1 << 1; + +/* + * The size of the block header exposed to used blocks is the size field. + * The prev_phys_block field is stored *inside* the previous free block. + */ +static const size_t block_header_overhead = sizeof(size_t); + +/* + * The size of the block header that overlaps the previous block, + * namely the size of prev_phys_block field. + */ +static const size_t block_header_overlap = sizeof(block_header_t *); + +/* User data starts directly after the size field in a used block. */ +static const size_t block_start_offset = + offsetof(block_header_t, size) + sizeof(size_t); + +/* + * A free block must be large enough to store its header minus the size of + * the metadata field, and no larger than the number of addressable + * bits for FL_INDEX. + */ +static const size_t block_size_min = + sizeof(block_header_t) - sizeof(size_t); // FIXME: metadata +static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX; + +/* The TLSF control structure. */ +typedef struct control_t { + /* Empty lists point at this block to indicate they are free. */ + block_header_t block_null; + + /* Bitmaps for free lists. */ + unsigned int fl_bitmap; + unsigned int sl_bitmap[FL_INDEX_COUNT]; + + /* Head of free lists. */ + block_header_t *blocks[FL_INDEX_COUNT][SL_INDEX_COUNT]; +} control_t; + +/* + * block_header_t member functions. + */ + +static size_t block_size(const block_header_t *block) +{ + return block->size & ~(block_header_free_bit | block_header_prev_free_bit); +} + +static void block_set_size(block_header_t *block, size_t size) +{ + const size_t oldsize = block->size; + block->size = + size | (oldsize & (block_header_free_bit | block_header_prev_free_bit)); +} + +static int block_is_last(const block_header_t *block) +{ + return block_size(block) == 0; +} + +static int block_is_free(const block_header_t *block) +{ + return tlsf_cast(int, block->size &block_header_free_bit); +} + +static void block_set_free(block_header_t *block) +{ + block->size |= block_header_free_bit; +} + +static void block_set_used(block_header_t *block) +{ + block->size &= ~block_header_free_bit; +} + +static int block_is_prev_free(const block_header_t *block) +{ + return tlsf_cast(int, block->size &block_header_prev_free_bit); +} + +static void block_set_prev_free(block_header_t *block) +{ + block->size |= block_header_prev_free_bit; +} + +static void block_set_prev_used(block_header_t *block) +{ + block->size &= ~block_header_prev_free_bit; +} + +static block_header_t *block_from_ptr(const void *ptr) +{ + return tlsf_cast(block_header_t *, + tlsf_cast(unsigned char *, ptr) - block_start_offset); +} + +static void *block_to_ptr(const block_header_t *block) +{ + return tlsf_cast(void *, + tlsf_cast(unsigned char *, block) + block_start_offset); +} + +/* Return location of next block after block of given size. */ +static block_header_t *offset_to_block(const void *ptr, ptrdiff_t size) +{ + return tlsf_cast(block_header_t *, + tlsf_cast(ptrdiff_t, ptr) + size - block_header_overlap); +} + +/* Return location of previous block. */ +static block_header_t *block_prev(const block_header_t *block) +{ + tlsf_assert(block_is_prev_free(block) && "previous block must be free"); + return block->prev_phys_block; +} + +/* Return location of next existing block. */ +static block_header_t *block_next(const block_header_t *block) +{ + block_header_t *next = + offset_to_block(block_to_ptr(block), block_size(block)); + tlsf_assert(!block_is_last(block)); + return next; +} + +/* Link a new block with its physical neighbor, return the neighbor. */ +static block_header_t *block_link_next(block_header_t *block) +{ + block_header_t *next = block_next(block); + next->prev_phys_block = block; + return next; +} + +static void block_mark_as_free(block_header_t *block) +{ + /* Link the block to the next block, first. */ + block_header_t *next = block_link_next(block); + block_set_prev_free(next); + block_set_free(block); +} + +static void block_mark_as_used(block_header_t *block) +{ + block_header_t *next = block_next(block); + block_set_prev_used(next); + block_set_used(block); +} + +static size_t align_up(size_t x, size_t align) +{ + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); + return (x + (align - 1)) & ~(align - 1); +} + +static size_t align_down(size_t x, size_t align) +{ + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); + return x - (x & (align - 1)); +} + +static void *align_ptr(const void *ptr, size_t align) +{ + const ptrdiff_t aligned = + (tlsf_cast(ptrdiff_t, ptr) + (align - 1)) & ~(align - 1); + tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two"); + return tlsf_cast(void *, aligned); +} + +/* + * Adjust an allocation size to be aligned to word size, and no smaller + * than internal minimum. +*/ +static size_t adjust_request_size(size_t size, size_t align) +{ + size_t adjust = 0; + if (size) { + const size_t aligned = align_up(size, align); + + /* aligned sized must not exceed block_size_max */ + if (aligned < block_size_max) { + adjust = tlsf_max(aligned, block_size_min); + } + } + return adjust; +} + +/* + * TLSF utility functions. In most cases, these are direct translations of + * the documentation found in the white paper. +*/ + +static void mapping_insert(size_t size, int *fli, int *sli) +{ + int fl, sl; + if (size < SMALL_BLOCK_SIZE) { + /* Store small blocks in first list. */ + fl = 0; + sl = tlsf_cast(int, size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT); + } else { + fl = tlsf_fls_sizet(size); + sl = tlsf_cast(int, size >> (fl - SL_INDEX_COUNT_LOG2)) ^ + (1 << SL_INDEX_COUNT_LOG2); + fl -= (FL_INDEX_SHIFT - 1); + } + *fli = fl; + *sli = sl; +} + +/* This version rounds up to the next block size (for allocations) */ +static void mapping_search(size_t size, int *fli, int *sli) +{ + if (size >= SMALL_BLOCK_SIZE) { + const size_t round = + (1 << (tlsf_fls_sizet(size) - SL_INDEX_COUNT_LOG2)) - 1; + size += round; + } + mapping_insert(size, fli, sli); +} + +static block_header_t *search_suitable_block(control_t *control, + int *fli, + int *sli) +{ + int fl = *fli; + int sl = *sli; + + /* + * First, search for a block in the list associated with the given + * fl/sl index. + */ + unsigned int sl_map = control->sl_bitmap[fl] & (((unsigned int)~0) << sl); + if (!sl_map) { + /* No block exists. Search in the next largest first-level list. */ + const unsigned int fl_map = + control->fl_bitmap & (((unsigned int)~0) << (fl + 1)); + if (!fl_map) { + /* No free blocks available, memory has been exhausted. */ + return NULL; + } + + fl = tlsf_ffs(fl_map); + *fli = fl; + sl_map = control->sl_bitmap[fl]; + } + tlsf_assert(sl_map && "internal error - second level bitmap is null"); + sl = tlsf_ffs(sl_map); + *sli = sl; + + /* Return the first block in the free list. */ + return control->blocks[fl][sl]; +} + +/* Remove a free block from the free list.*/ +static void remove_free_block(control_t *control, + block_header_t *block, + int fl, + int sl) +{ + block_header_t *prev = block->prev_free; + block_header_t *next = block->next_free; + tlsf_assert(prev && "prev_free field can not be null"); + tlsf_assert(next && "next_free field can not be null"); + next->prev_free = prev; + prev->next_free = next; + + /* If this block is the head of the free list, set new head. */ + if (control->blocks[fl][sl] == block) { + control->blocks[fl][sl] = next; + + /* If the new head is null, clear the bitmap. */ + if (next == &control->block_null) { + control->sl_bitmap[fl] &= ~(1U << sl); + + /* If the second bitmap is now empty, clear the fl bitmap. */ + if (!control->sl_bitmap[fl]) { + control->fl_bitmap &= ~(1U << fl); + } + } + } +} + +/* Insert a free block into the free block list. */ +static void insert_free_block(control_t *control, + block_header_t *block, + int fl, + int sl) +{ + block_header_t *current = control->blocks[fl][sl]; + tlsf_assert(current && "free list cannot have a null entry"); + tlsf_assert(block && "cannot insert a null entry into the free list"); + block->next_free = current; + block->prev_free = &control->block_null; + current->prev_free = block; + + tlsf_assert(block_to_ptr(block) == + align_ptr(block_to_ptr(block), ALIGN_SIZE) && + "block not aligned properly"); + /* + * Insert the new block at the head of the list, and mark the first- + * and second-level bitmaps appropriately. + */ + control->blocks[fl][sl] = block; + control->fl_bitmap |= (1U << fl); + control->sl_bitmap[fl] |= (1U << sl); +} + +/* Remove a given block from the free list. */ +static void block_remove(control_t *control, block_header_t *block) +{ + int fl, sl; + mapping_insert(block_size(block), &fl, &sl); + remove_free_block(control, block, fl, sl); +} + +/* Insert a given block into the free list. */ +static void block_insert(control_t *control, block_header_t *block) +{ + int fl, sl; + mapping_insert(block_size(block), &fl, &sl); + insert_free_block(control, block, fl, sl); +} + +static int block_can_split(block_header_t *block, size_t size) +{ + return block_size(block) >= sizeof(block_header_t) + size; +} + +/* Split a block into two, the second of which is free. */ +static block_header_t *block_split(block_header_t *block, size_t size) +{ + /* Calculate the amount of space left in the remaining block. */ + block_header_t *remaining = offset_to_block(block_to_ptr(block), size); + + const size_t remain_size = + block_size(block) - (size + block_header_overhead); + + tlsf_assert(block_to_ptr(remaining) == + align_ptr(block_to_ptr(remaining), ALIGN_SIZE) && + "remaining block not aligned properly"); + + tlsf_assert(block_size(block) == + remain_size + size + block_header_overhead); + block_set_size(remaining, remain_size); + tlsf_assert(block_size(remaining) >= block_size_min && + "block split with invalid size"); + + block_set_size(block, size); + block_mark_as_free(remaining); + + return remaining; +} + +/* Absorb a free block's storage into an adjacent previous free block. */ +static block_header_t *block_absorb(block_header_t *prev, block_header_t *block) +{ + tlsf_assert(!block_is_last(prev) && "previous block can't be last"); + /* Note: Leaves flags untouched. */ + prev->size += block_size(block) + block_header_overhead; + block_link_next(prev); + return prev; +} + +/* Merge a just-freed block with an adjacent previous free block. */ +static block_header_t *block_merge_prev(control_t *control, + block_header_t *block) +{ + if (block_is_prev_free(block)) { + block_header_t *prev = block_prev(block); + tlsf_assert(prev && "prev physical block can't be null"); + tlsf_assert(block_is_free(prev) && + "prev block is not free though marked as such"); + block_remove(control, prev); + block = block_absorb(prev, block); + } + + return block; +} + +/* Merge a just-freed block with an adjacent free block. */ +static block_header_t *block_merge_next(control_t *control, + block_header_t *block) +{ + block_header_t *next = block_next(block); + tlsf_assert(next && "next physical block can't be null"); + + if (block_is_free(next)) { + tlsf_assert(!block_is_last(block) && "previous block can't be last"); + block_remove(control, next); + block = block_absorb(block, next); + } + + return block; +} + +/* Trim any trailing block space off the end of a block, return to pool. */ +static void block_trim_free(control_t *control, + block_header_t *block, + size_t size) +{ + tlsf_assert(block_is_free(block) && "block must be free"); + if (block_can_split(block, size)) { + block_header_t *remaining_block = block_split(block, size); + block_link_next(block); + block_set_prev_free(remaining_block); + block_insert(control, remaining_block); + } +} + +/* Trim any trailing block space off the end of a used block, return to pool. */ +static void block_trim_used(control_t *control, + block_header_t *block, + size_t size) +{ + tlsf_assert(!block_is_free(block) && "block must be used"); + if (block_can_split(block, size)) { + /* If the next block is free, we must coalesce. */ + block_header_t *remaining_block = block_split(block, size); + block_set_prev_used(remaining_block); + + remaining_block = block_merge_next(control, remaining_block); + block_insert(control, remaining_block); + } +} + +/* If possible, create a trailing free block after trimming given block by size + */ +static block_header_t *block_trim_free_leading(control_t *control, + block_header_t *block, + size_t size) +{ + block_header_t *remaining_block = block; + if (block_can_split(block, size)) { + /* We want the 2nd block. */ + remaining_block = block_split(block, size - block_header_overhead); + block_set_prev_free(remaining_block); + + block_link_next(block); + block_insert(control, block); + } + + return remaining_block; +} + +static block_header_t *block_locate_free(control_t *control, size_t size) +{ + int fl = 0, sl = 0; + block_header_t *block = NULL; + + if (size) { + mapping_search(size, &fl, &sl); + /* + * mapping_search can futz with the size, so for excessively large + * sizes it can sometimes wind up with indices that are off the end + * of the block array. + * So, we protect against that here, since this is the only callsite of + * mapping_search. + * Note that we don't need to check sl, since it comes from a modulo + * operation that guarantees it's always in range. + */ + if (fl < FL_INDEX_COUNT) { + block = search_suitable_block(control, &fl, &sl); + } + } + + if (block) { + tlsf_assert(block_size(block) >= size); + remove_free_block(control, block, fl, sl); + } + + if (unlikely(block && !block->size)) + block = NULL; + + return block; +} + +static void *block_prepare_used(control_t *control, + block_header_t *block, + size_t size) +{ + void *p = NULL; + if (block) { + tlsf_assert(size && "size must be non-zero"); + block_trim_free(control, block, size); + block_mark_as_used(block); + p = block_to_ptr(block); + } + return p; +} + +/* Clear structure and point all empty lists at the null block. */ +static void control_construct(control_t *control) +{ + int i, j; + + control->block_null.next_free = &control->block_null; + control->block_null.prev_free = &control->block_null; + + control->fl_bitmap = 0; + for (i = 0; i < FL_INDEX_COUNT; ++i) { + control->sl_bitmap[i] = 0; + for (j = 0; j < SL_INDEX_COUNT; ++j) { + control->blocks[i][j] = &control->block_null; + } + } +} + +/* + * Debugging utilities. + */ +#ifdef TLSF_CONFIG_DEBUG + +typedef struct integrity_t { + int prev_status; + int status; +} integrity_t; + +#define tlsf_insist(x) \ + do { \ + tlsf_assert(x); \ + if (!(x)) \ + status--; \ + } while (0) + +static void integrity_walker(void *ptr, size_t size, int used, void *user) +{ + block_header_t *block = block_from_ptr(ptr); + integrity_t *integ = tlsf_cast(integrity_t *, user); + const int this_prev_status = block_is_prev_free(block) ? 1 : 0; + const int this_status = block_is_free(block) ? 1 : 0; + const size_t this_block_size = block_size(block); + + int status = 0; + (void)used; + tlsf_insist(integ->prev_status == this_prev_status && + "prev status incorrect"); + tlsf_insist(size == this_block_size && "block size incorrect"); + + integ->prev_status = this_status; + integ->status += status; +} + +int tlsf_check(tlsf_t tlsf) +{ + int i, j; + + control_t *control = tlsf_cast(control_t *, tlsf); + int status = 0; + + /* Check that the free lists and bitmaps are accurate. */ + for (i = 0; i < FL_INDEX_COUNT; ++i) { + for (j = 0; j < SL_INDEX_COUNT; ++j) { + const int fl_map = control->fl_bitmap & (1 << i); + const int sl_list = control->sl_bitmap[i]; + const int sl_map = sl_list & (1 << j); + const block_header_t *block = control->blocks[i][j]; + + /* Check that first- and second-level lists agree. */ + if (!fl_map) { + tlsf_insist(!sl_map && "second-level map must be null"); + } + + if (!sl_map) { + tlsf_insist(block == &control->block_null && + "block list must be null"); + continue; + } + + /* Check that there is at least one free block. */ + tlsf_insist(sl_list && "no free blocks in second-level map"); + tlsf_insist(block != &control->block_null && + "block should not be null"); + + while (block != &control->block_null) { + int fli, sli; + tlsf_insist(block_is_free(block) && "block should be free"); + tlsf_insist(!block_is_prev_free(block) && + "blocks should have coalesced"); + tlsf_insist(!block_is_free(block_next(block)) && + "blocks should have coalesced"); + tlsf_insist(block_is_prev_free(block_next(block)) && + "block should be free"); + tlsf_insist(block_size(block) >= block_size_min && + "block not minimum size"); + + mapping_insert(block_size(block), &fli, &sli); + tlsf_insist(fli == i && sli == j && + "block size indexed in wrong list"); + block = block->next_free; + } + } + } + + return status; +} + +#undef tlsf_insist + +static void default_walker(void *ptr, size_t size, int used, void *user) +{ + (void)user; + printf("\t%p %s size: %x (%p)\n", ptr, used ? "used" : "free", + (unsigned int)size, block_from_ptr(ptr)); +} + +void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void *user) +{ + tlsf_walker pool_walker = walker ? walker : default_walker; + block_header_t *block = offset_to_block(pool, 0); + + while (block && !block_is_last(block)) { + pool_walker(block_to_ptr(block), block_size(block), + !block_is_free(block), user); + block = block_next(block); + } +} + +size_t tlsf_block_size(void *ptr) +{ + size_t size = 0; + if (ptr) { + const block_header_t *block = block_from_ptr(ptr); + size = block_size(block); + } + return size; +} + +int tlsf_check_pool(pool_t pool) +{ + /* Check that the blocks are physically correct. */ + integrity_t integ = {0, 0}; + tlsf_walk_pool(pool, integrity_walker, &integ); + + return integ.status; +} + +#endif /* TLSF_CONFIG_DEBUG */ + +/* + * Size of the TLSF structures in a given memory block passed to + * tlsf_create, equal to the size of a control_t + */ +size_t tlsf_size(void) +{ + return sizeof(control_t); +} + +size_t tlsf_align_size(void) +{ + return ALIGN_SIZE; +} + +size_t tlsf_block_size_min(void) +{ + return block_size_min; +} + +size_t tlsf_block_size_max(void) +{ + return block_size_max; +} + +/* + * Overhead of the TLSF structures in a given memory block passed to + * tlsf_add_pool, equal to the overhead of a free block and the + * sentinel block. + */ +size_t tlsf_pool_overhead(void) +{ + return 2 * block_header_overhead; +} + +size_t tlsf_alloc_overhead(void) +{ + return block_header_overhead; +} + +pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes) +{ + block_header_t *block; + block_header_t *next; + + const size_t pool_overhead = tlsf_pool_overhead(); + const size_t pool_bytes = align_down(bytes - pool_overhead, ALIGN_SIZE); + + if (((ptrdiff_t)mem % ALIGN_SIZE) != 0) { + printf("tlsf_add_pool: Memory must be aligned by %u bytes.\n", + (unsigned int)ALIGN_SIZE); + return 0; + } + + if (pool_bytes < block_size_min || pool_bytes > block_size_max) { + printf( + "tlsf_add_pool: Memory size must be between %zu and %zu bytes.\n", + pool_overhead + block_size_min, pool_overhead + block_size_max); + return 0; + } + + /* + * Create the main free block. Offset the start of the block slightly + * so that the prev_phys_block field falls outside of the pool - + * it will never be used. + */ + block = offset_to_block(mem, 0); + block_set_size(block, pool_bytes); + block_set_free(block); + block_set_prev_used(block); + block_insert(tlsf_cast(control_t *, tlsf), block); + + /* Split the block to create a zero-size sentinel block. */ + next = block_link_next(block); + block_set_size(next, 0); + block_set_used(next); + block_set_prev_free(next); + + return mem; +} + +void tlsf_remove_pool(tlsf_t tlsf, pool_t pool) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + block_header_t *block = offset_to_block(pool, 0); + + int fl = 0, sl = 0; + + tlsf_assert(block_is_free(block) && "block should be free"); + tlsf_assert(!block_is_free(block_next(block)) && + "next block should not be free"); + tlsf_assert(block_size(block_next(block)) == 0 && + "next block size should be zero"); + + mapping_insert(block_size(block), &fl, &sl); + remove_free_block(control, block, fl, sl); +} + +/* + * TLSF main interface. + */ + +tlsf_t tlsf_create(void *mem) +{ + if (((ptrdiff_t)mem % ALIGN_SIZE) != 0) { + printf("tlsf_create: Memory must be aligned to %u bytes.\n", + (unsigned int)ALIGN_SIZE); + return NULL; + } + + control_construct(tlsf_cast(control_t *, mem)); + + return tlsf_cast(tlsf_t, mem); +} + +tlsf_t tlsf_create_with_pool(void *mem, size_t bytes) +{ + tlsf_t tlsf = tlsf_create(mem); + tlsf_add_pool(tlsf, (char *)mem + tlsf_size(), bytes - tlsf_size()); + return tlsf; +} + +void tlsf_destroy(tlsf_t tlsf) +{ + /* Nothing to do. */ + (void)tlsf; +} + +pool_t tlsf_get_pool(tlsf_t tlsf) +{ + return tlsf_cast(pool_t, (char *)tlsf + tlsf_size()); +} + +void *tlsf_malloc(tlsf_t tlsf, size_t size) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + block_header_t *block = block_locate_free(control, adjust); + return block_prepare_used(control, block, adjust); +} + +void *tlsf_memalign(tlsf_t tlsf, size_t align, size_t size) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + + /* + * We must allocate an additional minimum block size bytes so that if + * our free block will leave an alignment gap which is smaller, we can + * trim a leading free block and release it back to the pool. We must + * do this because the previous physical block is in use, therefore + * the prev_phys_block field is not valid, and we can't simply adjust + * the size of that block. + */ + const size_t gap_minimum = sizeof(block_header_t); + const size_t size_with_gap = + adjust_request_size(adjust + align + gap_minimum, align); + + /* + * If alignment is less than or equals base alignment, we're done. + * If we requested 0 bytes, return null, as tlsf_malloc(0) does. + */ + const size_t aligned_size = + (adjust && align > ALIGN_SIZE) ? size_with_gap : adjust; + + block_header_t *block = block_locate_free(control, aligned_size); + + /* This can't be a static assert. */ + tlsf_assert(sizeof(block_header_t) == + block_size_min + block_header_overhead); + + if (block) { + void *ptr = block_to_ptr(block); + void *aligned = align_ptr(ptr, align); + size_t gap = tlsf_cast( + size_t, tlsf_cast(ptrdiff_t, aligned) - tlsf_cast(ptrdiff_t, ptr)); + + /* If gap size is too small, offset to next aligned boundary. */ + if (gap && gap < gap_minimum) { + const size_t gap_remain = gap_minimum - gap; + const size_t offset = tlsf_max(gap_remain, align); + const void *next_aligned = + tlsf_cast(void *, tlsf_cast(ptrdiff_t, aligned) + offset); + + aligned = align_ptr(next_aligned, align); + gap = tlsf_cast(size_t, tlsf_cast(ptrdiff_t, aligned) - + tlsf_cast(ptrdiff_t, ptr)); + } + + if (gap) { + tlsf_assert(gap >= gap_minimum && "gap size too small"); + block = block_trim_free_leading(control, block, gap); + } + } + + return block_prepare_used(control, block, adjust); +} + +void tlsf_free(tlsf_t tlsf, void *ptr) +{ + if (unlikely(!ptr)) + return; + + control_t *control = tlsf_cast(control_t *, tlsf); + block_header_t *block = block_from_ptr(ptr); + tlsf_assert(!block_is_free(block) && "block already marked as free"); + block_mark_as_free(block); + block = block_merge_prev(control, block); + block = block_merge_next(control, block); + block_insert(control, block); +} + +/* + * The TLSF block information provides us with enough information to + * provide a reasonably intelligent implementation of realloc, growing or + * shrinking the currently allocated block as required. + * + * This routine handles the somewhat esoteric edge cases of realloc: + * - a non-zero size with a null pointer will behave like malloc + * - a zero size with a non-null pointer will behave like free + * - a request that cannot be satisfied will leave the original buffer + * untouched + * - an extended buffer size will leave the newly-allocated area with + * contents undefined + */ +void *tlsf_realloc(tlsf_t tlsf, void *ptr, size_t size) +{ + control_t *control = tlsf_cast(control_t *, tlsf); + void *p = NULL; + + /* Zero-size requests are treated as free. */ + if (ptr && size == 0) { + tlsf_free(tlsf, ptr); + } + /* Requests with NULL pointers are treated as malloc. */ + else if (!ptr) { + p = tlsf_malloc(tlsf, size); + } else { + block_header_t *block = block_from_ptr(ptr); + block_header_t *next = block_next(block); + + const size_t cursize = block_size(block); + const size_t combined = + cursize + block_size(next) + block_header_overhead; + const size_t adjust = adjust_request_size(size, ALIGN_SIZE); + + tlsf_assert(!block_is_free(block) && "block already marked as free"); + + /* + * If the next block is used, or when combined with the current + * block, does not offer enough space, we must reallocate and copy. + */ + if (adjust > cursize && (!block_is_free(next) || adjust > combined)) { + p = tlsf_malloc(tlsf, size); + if (p) { + const size_t minsize = tlsf_min(cursize, size); + memcpy(p, ptr, minsize); + tlsf_free(tlsf, ptr); + } + } else { + /* Do we need to expand to the next block? */ + if (adjust > cursize) { + block_merge_next(control, block); + block_mark_as_used(block); + } + + /* Trim the resulting block and return the original pointer. */ + block_trim_used(control, block, adjust); + p = ptr; + } + } + + return p; +} diff --git a/tlsf/tlsf.h b/tlsf/tlsf.h new file mode 100644 index 0000000..74a6b56 --- /dev/null +++ b/tlsf/tlsf.h @@ -0,0 +1,83 @@ +#ifndef INCLUDED_tlsf +#define INCLUDED_tlsf + +/* + * Two Level Segregated Fit memory allocator. + * + * Copyright (c) 2006-2008, 2011, 2014 Matthew Conte. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the copyright holder nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL MATTHEW CONTE BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +/* tlsf_t: a TLSF structure. Can contain 1 to N pools. */ +/* pool_t: a block of memory that TLSF can manage. */ +typedef void *tlsf_t; +typedef void *pool_t; + +/* Create/destroy a memory pool. */ +tlsf_t tlsf_create(void *mem); +tlsf_t tlsf_create_with_pool(void *mem, size_t bytes); +void tlsf_destroy(tlsf_t tlsf); +pool_t tlsf_get_pool(tlsf_t tlsf); + +/* Add/remove memory pools. */ +pool_t tlsf_add_pool(tlsf_t tlsf, void *mem, size_t bytes); +void tlsf_remove_pool(tlsf_t tlsf, pool_t pool); + +/* malloc/memalign/realloc/free replacements. */ +void *tlsf_malloc(tlsf_t tlsf, size_t bytes); +void *tlsf_memalign(tlsf_t tlsf, size_t align, size_t bytes); +void *tlsf_realloc(tlsf_t tlsf, void *ptr, size_t size); +void tlsf_free(tlsf_t tlsf, void *ptr); + +/* Returns internal block size, not original request size */ +size_t tlsf_block_size(void *ptr); + +/* Overheads/limits of internal structures. */ +size_t tlsf_size(void); +size_t tlsf_align_size(void); +size_t tlsf_block_size_min(void); +size_t tlsf_block_size_max(void); +size_t tlsf_pool_overhead(void); +size_t tlsf_alloc_overhead(void); + +/* Debugging. */ +#ifdef TLSF_CONFIG_DEBUG +typedef void (*tlsf_walker)(void *ptr, size_t size, int used, void *user); +void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void *user); +/* Returns nonzero if any internal consistency check fails. */ +int tlsf_check(tlsf_t tlsf); +int tlsf_check_pool(pool_t pool); +#endif /* TLSF_CONFIG_DEBUG */ + +#if defined(__cplusplus) +}; +#endif + +#endif diff --git a/tlsf/tlsf_utils.h b/tlsf/tlsf_utils.h new file mode 100644 index 0000000..144e07f --- /dev/null +++ b/tlsf/tlsf_utils.h @@ -0,0 +1,202 @@ +#ifndef INCLUDED_tlsf_utils +#define INCLUDED_tlsf_utils + +#include + +/* +** Architecture-specific bit manipulation routines. +** +** TLSF achieves O(1) cost for malloc and free operations by limiting +** the search for a free block to a free list of guaranteed size +** adequate to fulfill the request, combined with efficient free list +** queries using bitmasks and architecture-specific bit-manipulation +** routines. +** +** Most modern processors provide instructions to count leading zeroes +** in a word, find the lowest and highest set bit, etc. These +** specific implementations will be used when available, falling back +** to a reasonably efficient generic implementation. +** +** NOTE: TLSF spec relies on ffs/fls returning value 0..31. +** ffs/fls return 1-32 by default, returning 0 for error. +*/ + +/* +** Detect whether or not we are building for a 32- or 64-bit (LP/LLP) +** architecture. There is no reliable portable method at compile-time. +*/ +#if defined(__alpha__) || defined(__ia64__) || defined(__x86_64__) || \ + defined(_WIN64) || defined(__LP64__) || defined(__LLP64__) || \ + defined(__aarch64__) +#define TLSF_64BIT +#endif + +#if defined(__GNUC__) + +/* count leading zeroes: + * clz(0) == 32, clz(0xf) == 28, clz(1 << 31) == 0 + */ +static inline int clz(uint32_t x) +{ + return x ? __builtin_clz(x) : sizeof(x) * 8; +} + +/* integer binary logarithm (rounding down): + * log2(0) == -1, log2(5) == 2 + */ +static inline int __log2(uint32_t x) +{ + return sizeof(x) * 8 - clz(x) - 1; +} + +/* find first set: + * __ffs(1) == 0, __ffs(0) == -1, __ffs(1<<31) == 31 + */ +static inline int tlsf_ffs(unsigned int word) +{ + return __log2(word & (uint32_t)(-(uint32_t)word)); +} + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = word ? 32 - __builtin_clz(word) : 0; + return bit - 1; +} + +#elif defined(_MSC_VER) && (_MSC_VER >= 1400) && \ + (defined(_M_IX86) || defined(_M_X64)) +/* Microsoft Visual C++ support on x86/X64 architectures. */ + +#include + +#pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanForward) + +static inline int tlsf_fls(unsigned int word) +{ + unsigned long index; + return _BitScanReverse(&index, word) ? index : -1; +} + +static inline int tlsf_ffs(unsigned int word) +{ + unsigned long index; + return _BitScanForward(&index, word) ? index : -1; +} + +#elif defined(_MSC_VER) && defined(_M_PPC) +/* Microsoft Visual C++ support on PowerPC architectures. */ + +#include + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = 32 - _CountLeadingZeros(word); + return bit - 1; +} + +static inline int tlsf_ffs(unsigned int word) +{ + const unsigned int reverse = word & (~word + 1); + const int bit = 32 - _CountLeadingZeros(reverse); + return bit - 1; +} + +#elif defined(__ARMCC_VERSION) +/* RealView Compilation Tools for ARM */ + +static inline int tlsf_ffs(unsigned int word) +{ + const unsigned int reverse = word & (~word + 1); + const int bit = 32 - __clz(reverse); + return bit - 1; +} + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = word ? 32 - __clz(word) : 0; + return bit - 1; +} + +#elif defined(__ghs__) +/* Green Hills support for PowerPC */ + +#include + +static inline int tlsf_ffs(unsigned int word) +{ + const unsigned int reverse = word & (~word + 1); + const int bit = 32 - __CLZ32(reverse); + return bit - 1; +} + +static inline int tlsf_fls(unsigned int word) +{ + const int bit = word ? 32 - __CLZ32(word) : 0; + return bit - 1; +} + +#else +/* Fall back to generic implementation. */ + +static inline int tlsf_fls_generic(unsigned int word) +{ + int bit = 32; + + if (!word) + bit -= 1; + if (!(word & 0xffff0000)) { + word <<= 16; + bit -= 16; + } + if (!(word & 0xff000000)) { + word <<= 8; + bit -= 8; + } + if (!(word & 0xf0000000)) { + word <<= 4; + bit -= 4; + } + if (!(word & 0xc0000000)) { + word <<= 2; + bit -= 2; + } + if (!(word & 0x80000000)) { + word <<= 1; + bit -= 1; + } + + return bit; +} + +/* Implement ffs in terms of fls. */ +static inline int tlsf_ffs(unsigned int word) +{ + return tlsf_fls_generic(word & (~word + 1)) - 1; +} + +static inline int tlsf_fls(unsigned int word) +{ + return tlsf_fls_generic(word) - 1; +} + +#endif + +/* Possibly 64-bit version of tlsf_fls. */ +#if defined(TLSF_64BIT) +static inline int tlsf_fls_sizet(size_t size) +{ + int high = (int)(size >> 32); + int bits = 0; + if (high) { + bits = 32 + tlsf_fls(high); + } else { + bits = tlsf_fls((int)size & 0xffffffff); + } + return bits; +} +#else +#define tlsf_fls_sizet tlsf_fls +#endif + +#endif /* INCLUDED_tlsf_utils */