From aed1b20e9f134bf9f7de6f064de8f592ba51886f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Oct 2026 12:10:04 +0100 Subject: [PATCH 1/2] core:mem/virtual: Allow for shrinking in-place with `Arena` --- core/mem/virtual/arena.odin | 38 +++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/core/mem/virtual/arena.odin b/core/mem/virtual/arena.odin index 45ed98298..6d810d6b1 100644 --- a/core/mem/virtual/arena.odin +++ b/core/mem/virtual/arena.odin @@ -347,7 +347,7 @@ arena_allocator_proc :: proc(allocator_data: rawptr, mode: mem.Allocator_Mode, switch { case old_data == nil: return arena_alloc(arena, size, alignment, location) - case size == old_size: + case size == old_size && uintptr(old_data) & uintptr(alignment-1) == 0: // return old memory data = old_data[:size] return @@ -359,27 +359,41 @@ arena_allocator_proc :: proc(allocator_data: rawptr, mode: mem.Allocator_Mode, sync.mutex_guard(&arena.mutex) if uintptr(old_data) & uintptr(alignment-1) == 0 { - if size < old_size { - // shrink data in-place - data = old_data[:size] - // sanitizer.address_poison(old_data[size:old_size]) - return - } - if block := arena.curr_block; block != nil { start := uint(uintptr(old_memory)) - uint(uintptr(block.base)) old_end := start + old_size new_end := start + size if start < old_end && old_end == block.used && new_end <= block.reserved { - // grow data in-place, adjusting next allocation - prev_used := block.used - _ = alloc_from_memory_block(block, new_end - old_end, 1, default_commit_size=arena.default_commit_size) or_return - arena.total_used += block.used - prev_used + switch { + case size > old_size: + // Grow data in-place by adjusting the next allocation + prev_used := block.used + _ = alloc_from_memory_block(block, new_end - old_end, 1, default_commit_size=arena.default_commit_size) or_return + arena.total_used += block.used - prev_used + + case arena.temp_count == 0: + // Shrink data in-place by giving the tail back to the block + // + // NOTE(bill): This is not done while an `Arena_Temp` is active. + // It may have saved a position inside the tail. + // The memory past `used` is kept zeroed because `.Alloc` relies on it. + mem.zero_slice(block.base[new_end:old_end]) + // sanitizer.address_poison(block.base[new_end:old_end]) + block.used = new_end + arena.total_used -= old_size - size + } data = block.base[start:new_end] // sanitizer.address_unpoison(data) return } } + + if size < old_size { + // Allow for shrinking data in-place + data = old_data[:size] + // sanitizer.address_poison(old_data[size:old_size]) + return + } } new_memory := arena_alloc_unguarded(arena, size, alignment, location) or_return From 64872e01b67ab0787fb807de1ab0fbc4b897c6f8 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Oct 2026 12:13:28 +0100 Subject: [PATCH 2/2] core:mem: Resize last allocations in place and general bug fixes --- core/mem/allocators.odin | 219 +++++++++- core/mem/rollback_stack_allocator.odin | 7 +- core/mem/tlsf/tlsf_internal.odin | 8 +- tests/core/mem/test_mem_resize.odin | 543 +++++++++++++++++++++++++ 4 files changed, 758 insertions(+), 19 deletions(-) create mode 100644 tests/core/mem/test_mem_resize.odin diff --git a/core/mem/allocators.odin b/core/mem/allocators.odin index 57a360464..b95859446 100644 --- a/core/mem/allocators.odin +++ b/core/mem/allocators.odin @@ -289,6 +289,145 @@ arena_free_all :: proc(a: ^Arena) { // sanitizer.address_poison(a.data) } +/* +Resize an allocation owned by an arena. + +This procedure resizes a memory region defined by its location `old_memory` and its size `old_size` to have a size `size` and alignment `alignment`. +The newly allocated memory, if any, is zero-initialized. + +If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size, +aligned on a boundary specified by `alignment`. + +The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks, +unless a temporary memory region is active. Any other allocation shrinks in place, and moves to grow. + +This procedure returns the pointer to the resized memory region. +*/ +@(require_results) +arena_resize :: proc( + a: ^Arena, + old_memory: rawptr, + old_size: int, + size: int, + alignment := DEFAULT_ALIGNMENT, + loc := #caller_location, +) -> (rawptr, Allocator_Error) { + bytes, err := arena_resize_bytes(a, byte_slice(old_memory, old_size), size, alignment, loc) + return raw_data(bytes), err +} + +/* +Resize an allocation owned by an arena. + +This procedure resizes a memory region specified by `old_data` to have a size `size` and alignment `alignment`. +The newly allocated memory, if any, is zero-initialized. + +If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size, +aligned on a boundary specified by `alignment`. + +The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks, unless a temporary memory region is active. +Any other allocation shrinks in place and moves to grow. + +This procedure returns the slice of the resized memory region. +*/ +@(require_results) +arena_resize_bytes :: proc( + a: ^Arena, + old_data: []byte, + size: int, + alignment := DEFAULT_ALIGNMENT, + loc := #caller_location, +) -> ([]byte, Allocator_Error) { + bytes, err := arena_resize_bytes_non_zeroed(a, old_data, size, alignment, loc) + if bytes != nil && size > len(old_data) { + zero_slice(bytes[len(old_data):]) + } + return bytes, err +} + +/* +Resize an allocation owned by an arena, without zero-initialization. + +This procedure resizes a memory region defined by its location `old_memory` and its size `old_size` to have a size `size` and alignment `alignment`. +The newly allocated memory, if any, is not explicitly zero-initialized. + +If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size, +aligned on a boundary specified by `alignment`. + +The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks, +unless a temporary memory region is active. Any other allocation shrinks in place, and moves to grow. + +This procedure returns the pointer to the resized memory region. +*/ +@(require_results) +arena_resize_non_zeroed :: proc( + a: ^Arena, + old_memory: rawptr, + old_size: int, + size: int, + alignment := DEFAULT_ALIGNMENT, + loc := #caller_location, +) -> (rawptr, Allocator_Error) { + bytes, err := arena_resize_bytes_non_zeroed(a, byte_slice(old_memory, old_size), size, alignment, loc) + return raw_data(bytes), err +} + +/* +Resize an allocation owned by an arena, without zero-initialization. + +This procedure resizes a memory region specified by `old_data` to have a size `size` and alignment `alignment`. +The newly allocated memory, if any, is not explicitly zero-initialized. + +If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size, +aligned on a boundary specified by `alignment`. + +The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks, +unless a temporary memory region is active. Any other allocation shrinks in place, and moves to grow. + +This procedure returns the slice of the resized memory region. +*/ +@(require_results) +arena_resize_bytes_non_zeroed :: proc( + a: ^Arena, + old_data: []byte, + size: int, + alignment := DEFAULT_ALIGNMENT, + loc := #caller_location, +) -> ([]byte, Allocator_Error) { + old_memory := raw_data(old_data) + old_size := len(old_data) + if old_memory == nil { + return arena_alloc_bytes_non_zeroed(a, size, alignment, loc) + } + if is_aligned(old_memory, alignment) { + begin := uintptr(raw_data(a.data)) + old_ptr := uintptr(old_memory) + is_last := begin <= old_ptr && old_ptr+uintptr(old_size) == begin+uintptr(a.offset) + switch { + case is_last && int(old_ptr-begin)+size <= len(a.data): + // NOTE(bill): A temporary memory region may have saved an offset inside the tail; therefore keep the tail while one is active. + if size > old_size || a.temp_count == 0 { + // sanitizer.address_poison(old_data[size:]) + a.offset = int(old_ptr-begin) + size + a.peak_used = max(a.peak_used, a.offset) + } + result := byte_slice(old_memory, size) + // sanitizer.address_unpoison(result) + return result, nil + + case size <= old_size: + // sanitizer.address_poison(old_data[size:]) + return byte_slice(old_memory, size), nil + } + } + data, err := arena_alloc_bytes_non_zeroed(a, size, alignment, loc) + if err == nil { + runtime.copy(data, old_data) + // sanitizer.address_poison(old_data) + } + return data, err +} + arena_allocator_proc :: proc( allocator_data: rawptr, mode: Allocator_Mode, @@ -309,9 +448,9 @@ arena_allocator_proc :: proc( case .Free_All: arena_free_all(arena) case .Resize: - return default_resize_bytes_align(byte_slice(old_memory, old_size), size, alignment, arena_allocator(arena), loc) + return arena_resize_bytes(arena, byte_slice(old_memory, old_size), size, alignment, loc) case .Resize_Non_Zeroed: - return default_resize_bytes_align_non_zeroed(byte_slice(old_memory, old_size), size, alignment, arena_allocator(arena), loc) + return arena_resize_bytes_non_zeroed(arena, byte_slice(old_memory, old_size), size, alignment, loc) case .Query_Features: set := (^Allocator_Mode_Set)(old_memory) if set != nil { @@ -607,8 +746,7 @@ scratch_free :: proc(s: ^Scratch, ptr: rawptr, loc := #caller_location) -> Alloc } if len(s.leaked_allocations) != 0 { for data, i in s.leaked_allocations { - ptr := raw_data(data) - if ptr == ptr { + if raw_data(data) == ptr { free_bytes(data, s.backup_allocator, loc) ordered_remove(&s.leaked_allocations, i, loc) return nil @@ -686,7 +824,7 @@ scratch_resize_bytes :: proc( ) -> ([]byte, Allocator_Error) { bytes, err := scratch_resize_bytes_non_zeroed(s, old_data, size, alignment, loc) if bytes != nil && size > len(old_data) { - zero_slice(bytes[size:]) + zero_slice(bytes[len(old_data):]) } return bytes, err } @@ -753,6 +891,9 @@ scratch_resize_bytes_non_zeroed :: proc( } scratch_init(s, DEFAULT_BACKING_SIZE) } + if old_memory == nil { + return scratch_alloc_bytes_non_zeroed(s, size, alignment, loc) + } begin := uintptr(raw_data(s.data)) end := begin + uintptr(len(s.data)) old_ptr := uintptr(old_memory) @@ -761,7 +902,7 @@ scratch_resize_bytes_non_zeroed :: proc( // // Also, the alignments must match, otherwise we must re-allocate to // guarantee the user's request. - if s.prev_allocation == old_memory && is_aligned(old_memory, alignment) && old_ptr+uintptr(size) < end { + if s.prev_allocation == old_memory && is_aligned(old_memory, alignment) && old_ptr+uintptr(size) <= end { // ensure_not_poisoned(old_data) // sanitizer.address_poison(old_memory) s.curr_offset = int(old_ptr-begin)+size @@ -1172,7 +1313,8 @@ stack_resize_bytes_non_zeroed :: proc( } header := (^Stack_Allocation_Header)(curr_addr - size_of(Stack_Allocation_Header)) old_offset := int(curr_addr - uintptr(header.padding) - uintptr(raw_data(s.data))) - if old_offset != header.prev_offset { + if old_offset != s.prev_offset { + // Not the last allocation, so it cannot be resized in place. data, err := stack_alloc_bytes_non_zeroed(s, size, alignment, loc) if err == nil { runtime.copy(data, byte_slice(old_memory, old_size)) @@ -1183,10 +1325,12 @@ stack_resize_bytes_non_zeroed :: proc( old_memory_size := uintptr(s.curr_offset) - (curr_addr - start) assert(old_memory_size == uintptr(old_size)) diff := size - old_size + if s.curr_offset + diff > len(s.data) { + return nil, .Out_Of_Memory + } s.curr_offset += diff // works for smaller sizes too - if diff > 0 { - zero(rawptr(curr_addr + uintptr(diff)), diff) - } else { + s.peak_used = max(s.peak_used, s.curr_offset) + if diff < 0 { // sanitizer.address_poison(old_data[size:]) } result := byte_slice(old_memory, size) @@ -1592,6 +1736,20 @@ small_stack_resize_bytes_non_zeroed :: proc( // sanitizer.address_unpoison(result) return result, nil } + is_last := curr_addr+uintptr(old_size) == start+uintptr(s.offset) + switch { + case is_last && int(curr_addr-start)+size <= len(s.data): + // sanitizer.address_poison(old_data[size:]) + s.offset = int(curr_addr-start) + size + s.peak_used = max(s.peak_used, s.offset) + result := byte_slice(old_memory, size) + // sanitizer.address_unpoison(result) + return result, nil + + case size < old_size: + // sanitizer.address_poison(old_data[size:]) + return byte_slice(old_memory, size), nil + } data, err := small_stack_alloc_bytes_non_zeroed(s, size, alignment, loc) if err == nil { runtime.copy(data, byte_slice(old_memory, old_size)) @@ -2002,12 +2160,34 @@ dynamic_arena_resize_bytes_non_zeroed :: proc( } old_memory := raw_data(old_data) old_size := len(old_data) - if old_size >= size { - // sanitizer.address_poison(old_data[size:]) - return byte_slice(old_memory, size), nil + if is_aligned(old_memory, alignment) { + // No information is kept about allocations in this allocator, but the + // last one in the current block ends less than `minimum_alignment` + // bytes before `current_pos`, as any later one takes at least that + // many bytes. It can be resized in place. + old_end := uintptr(old_memory) + uintptr(old_size) + curr_pos := uintptr(a.current_pos) + block_end := curr_pos + uintptr(a.bytes_left) + if old_size > 0 && + uintptr(a.current_block) <= uintptr(old_memory) && + old_end <= curr_pos && + curr_pos-old_end < uintptr(a.minimum_alignment) { + new_pos := uintptr(old_memory) + uintptr(align_formula(size, max(a.minimum_alignment, alignment))) + if new_pos <= block_end { + // sanitizer.address_poison(old_data[size:]) + a.current_pos = rawptr(new_pos) + a.bytes_left = int(block_end - new_pos) + result := byte_slice(old_memory, size) + // sanitizer.address_unpoison(result) + return result, nil + } + } + if old_size >= size { + // sanitizer.address_poison(old_data[size:]) + return byte_slice(old_memory, size), nil + } } - // No information is kept about allocations in this allocator, thus we - // cannot truly resize anything and must reallocate. + // Any other allocation cannot truly be resized, and must be reallocated. data, err := dynamic_arena_alloc_bytes_non_zeroed(a, size, alignment, loc) if err == nil { runtime.copy(data, byte_slice(old_memory, old_size)) @@ -2491,6 +2671,11 @@ compat_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, return rra.parent.procedure(rra.parent.data, mode, orig_size, header.alignment, orig_ptr, orig_size, location) case .Resize, .Resize_Non_Zeroed: + if old_memory == nil { + alloc_mode := Allocator_Mode.Alloc if mode == .Resize else .Alloc_Non_Zeroed + return compat_allocator_proc(allocator_data, alloc_mode, size, alignment, nil, 0, location) + } + header := get_unpoisoned_header(old_memory) orig_a := max(header.alignment, size_of(Header)) orig_ptr := rawptr(uintptr(old_memory)-uintptr(orig_a)) @@ -2503,6 +2688,10 @@ compat_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, assert(size >= 0, "overflow") allocation := rra.parent.procedure(rra.parent.data, mode, req_size, new_alignment, orig_ptr, orig_size, location) or_return + if a != orig_a { + // The header grew with the alignment, so the data must move up after it. + runtime.copy(allocation[a:], allocation[orig_a:][:min(header.size, size)]) + } #no_bounds_check data = allocation[a:] ([^]Header)(raw_data(data))[-1] = { diff --git a/core/mem/rollback_stack_allocator.odin b/core/mem/rollback_stack_allocator.odin index 3c98135de..7a0f09f27 100644 --- a/core/mem/rollback_stack_allocator.odin +++ b/core/mem/rollback_stack_allocator.odin @@ -326,12 +326,15 @@ rb_resize_bytes_non_zeroed :: proc( assert(size >= 0, "Size must be positive or zero.", loc) assert(old_size >= 0, "Old size must be positive or zero.", loc) assert(is_power_of_two(cast(uintptr)alignment), "Alignment must be a power of two.", loc) - if ptr != nil { + if ptr == nil { + return rb_alloc_bytes_non_zeroed(stack, size, alignment, loc) + } + if is_aligned(ptr, alignment) { if block, _, ok := rb_find_last_alloc(stack, ptr); ok { // `block.offset` should never underflow because it is contingent // on `old_size` in the first place, assuming sane arguments. assert(block.offset >= cast(uintptr)old_size, "Rollback Stack Allocator received invalid `old_size`.") - if block.offset + cast(uintptr)size - cast(uintptr)old_size < cast(uintptr)len(block.buffer) { + if block.offset + cast(uintptr)size - cast(uintptr)old_size <= cast(uintptr)len(block.buffer) { // Prevent singleton allocations from fragmenting by forbidding // them to shrink, removing the possibility of overflow bugs. if len(block.buffer) <= stack.block_size { diff --git a/core/mem/tlsf/tlsf_internal.odin b/core/mem/tlsf/tlsf_internal.odin index 802c4a3fd..2cc3bd6b4 100644 --- a/core/mem/tlsf/tlsf_internal.odin +++ b/core/mem/tlsf/tlsf_internal.odin @@ -306,7 +306,9 @@ resize :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, align min_size := min(curr_size, new_size, old_size) - if adjust > curr_size && (!block_is_free(next) || adjust > combined) { + // NOTE(bill): A stricter alignment than the block has cannot be met in place. + misaligned := uintptr(ptr) % uintptr(max(ALIGN_SIZE, alignment)) != 0 + if misaligned || adjust > curr_size && (!block_is_free(next) || adjust > combined) { res = alloc_bytes(control, new_size, alignment) or_return if res != nil { copy(res, ([^]byte)(ptr)[:min_size]) @@ -351,7 +353,9 @@ resize_non_zeroed :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: min_size := min(curr_size, new_size, old_size) - if adjust > curr_size && (!block_is_free(next) || adjust > combined) { + // NOTE(bill): A stricter alignment than the block has cannot be met in place. + misaligned := uintptr(ptr) % uintptr(max(ALIGN_SIZE, alignment)) != 0 + if misaligned || adjust > curr_size && (!block_is_free(next) || adjust > combined) { res = alloc_bytes_non_zeroed(control, new_size, alignment) or_return if res != nil { copy(res, ([^]byte)(ptr)[:min_size]) diff --git a/tests/core/mem/test_mem_resize.odin b/tests/core/mem/test_mem_resize.odin new file mode 100644 index 000000000..22c79cc92 --- /dev/null +++ b/tests/core/mem/test_mem_resize.odin @@ -0,0 +1,543 @@ +package test_core_mem + +import "core:mem" +import "core:mem/tlsf" +import "core:mem/virtual" +import "core:testing" + +@(require_results) +count_byte :: proc(data: []byte, value: byte) -> (n: int) { + for b in data { + if b == value { + n += 1 + } + } + return +} + +/* +Grow, grow without zeroing, then shrink one allocation, checking that the old +bytes survive each step and that the zeroing resize clears the new tail. + +Fill the backing memory with non-zero bytes first where possible, so that a +tail which is not zeroed shows up. +*/ +expect_resize_keeps_bytes :: proc(t: ^testing.T, allocator: mem.Allocator, loc := #caller_location) { + data, err := mem.alloc_bytes(100, 16, allocator) + if !testing.expect_value(t, err, nil, loc) { + return + } + data = data[:100] // TLSF hands back the whole block + mem.set(raw_data(data), 0xAA, len(data)) + + data, err = mem.resize_bytes(data, 150, 16, allocator) + if !testing.expect_value(t, err, nil, loc) { + return + } + testing.expect_value(t, count_byte(data[:100], 0xAA), 100, loc) + testing.expect_value(t, count_byte(data[100:], 0), 50, loc) + + data, err = mem.resize_bytes_non_zeroed(data, 200, 16, allocator) + if !testing.expect_value(t, err, nil, loc) { + return + } + testing.expect_value(t, count_byte(data[:100], 0xAA), 100, loc) + testing.expect_value(t, count_byte(data[100:150], 0), 50, loc) + + data, err = mem.resize_bytes(data, 50, 16, allocator) + if !testing.expect_value(t, err, nil, loc) { + return + } + testing.expect_value(t, count_byte(data, 0xAA), 50, loc) + _ = mem.free_bytes(data, allocator) +} + +// Allocate 48 bytes off a 64-byte boundary, holding their own indices. +@(require_results) +alloc_off_64 :: proc(t: ^testing.T, allocator: mem.Allocator, loc := #caller_location) -> (data: []byte, ok: bool) { + for i in 0..<8 { + data, _ = mem.alloc_bytes(48, 16, allocator) + if data != nil && !mem.is_aligned(raw_data(data), 64) { + data = data[:48] + for &b, j in data { + b = byte(j) + } + return data, true + } + // Shift the next allocation along, by a different amount each time. + _, _ = mem.alloc_bytes(16*(i+1), 16, allocator) + } + testing.expect(t, false, "found no allocation off a 64-byte boundary", loc=loc) + return nil, false +} + +// Resizing to a stricter alignment than the allocation has must move it, keeping its bytes. +expect_resize_realigns :: proc(t: ^testing.T, allocator: mem.Allocator, loc := #caller_location) { + for size in ([]int{48, 32, 96}) { + data := alloc_off_64(t, allocator, loc) or_break + + resized, err := mem.resize_bytes(data, size, 64, allocator) + if !testing.expect_value(t, err, nil, loc) { + continue + } + testing.expectf(t, mem.is_aligned(raw_data(resized), 64), "resizing to %d bytes left them at %p", size, raw_data(resized), loc=loc) + + kept := 0 + for b, i in resized[:min(size, 48)] { + if b == byte(i) { + kept += 1 + } + } + testing.expectf(t, kept == min(size, 48), "resizing to %d bytes kept %d of the old bytes", size, kept, loc=loc) + _ = mem.free_bytes(resized, allocator) + } +} + +@(test) +test_stack_resize :: proc(t: ^testing.T) { + buf: [4096]byte + mem.set(&buf, 0xCC, len(buf)) + + s: mem.Stack + mem.stack_init(&s, buf[:]) + expect_resize_keeps_bytes(t, mem.stack_allocator(&s)) + expect_resize_realigns(t, mem.stack_allocator(&s)) +} + +@(test) +test_stack_resize_out_of_memory :: proc(t: ^testing.T) { + // The rest of `buf` takes any write past the end of the stack. + buf: [8192]byte + s: mem.Stack + mem.stack_init(&s, buf[:1024]) + allocator := mem.stack_allocator(&s) + + data, _ := mem.alloc_bytes(512, 16, allocator) + offset := s.curr_offset + + grown, err := mem.resize_bytes(data, 4096, 16, allocator) + testing.expect_value(t, err, mem.Allocator_Error.Out_Of_Memory) + testing.expect(t, grown == nil) + testing.expect_value(t, s.curr_offset, offset) +} + +@(test) +test_stack_resize_last_allocation :: proc(t: ^testing.T) { + buf: [1024]byte + s: mem.Stack + mem.stack_init(&s, buf[:]) + allocator := mem.stack_allocator(&s) + + _, _ = mem.alloc_bytes(100, 16, allocator) + last, _ := mem.alloc_bytes(100, 16, allocator) + offset := s.curr_offset + + grown, err := mem.resize_bytes(last, 150, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place") + testing.expect_value(t, s.curr_offset, offset+50) + testing.expect_value(t, s.peak_used, offset+50) + + shrunk, shrunk_err := mem.resize_bytes(grown, 20, 16, allocator) + testing.expect_value(t, shrunk_err, nil) + testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place") + testing.expect_value(t, s.curr_offset, offset-80) +} + +@(test) +test_stack_resize_earlier_allocation :: proc(t: ^testing.T) { + buf: [1024]byte + s: mem.Stack + mem.stack_init(&s, buf[:]) + allocator := mem.stack_allocator(&s) + + first, _ := mem.alloc_bytes(100, 16, allocator) + last, _ := mem.alloc_bytes(100, 16, allocator) + mem.set(raw_data(first), 0xAA, len(first)) + mem.set(raw_data(last), 0xBB, len(last)) + + grown, err := mem.resize_bytes(first, 150, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(grown) != raw_data(first), "an earlier allocation grew in place over a later one") + testing.expect_value(t, count_byte(grown[:100], 0xAA), 100) + testing.expect_value(t, count_byte(grown[100:], 0), 50) + testing.expect_value(t, count_byte(last, 0xBB), 100) +} + +@(test) +test_small_stack_resize :: proc(t: ^testing.T) { + buf: [4096]byte + mem.set(&buf, 0xCC, len(buf)) + + s: mem.Small_Stack + mem.small_stack_init(&s, buf[:]) + expect_resize_keeps_bytes(t, mem.small_stack_allocator(&s)) + expect_resize_realigns(t, mem.small_stack_allocator(&s)) +} + +@(test) +test_scratch_resize :: proc(t: ^testing.T) { + s: mem.Scratch + mem.scratch_init(&s, 4096) + defer mem.scratch_destroy(&s) + mem.set(raw_data(s.data), 0xCC, len(s.data)) + + expect_resize_keeps_bytes(t, mem.scratch_allocator(&s)) + expect_resize_realigns(t, mem.scratch_allocator(&s)) +} + +@(test) +test_scratch_resize_leaked_allocation :: proc(t: ^testing.T) { + s: mem.Scratch + mem.scratch_init(&s, 64) + defer mem.scratch_destroy(&s) + allocator := mem.scratch_allocator(&s) + + // Both are too big for the buffer, so they come from the backup allocator. + first, _ := mem.alloc_bytes(128, 16, allocator) + last, _ := mem.alloc_bytes(128, 16, allocator) + + grown, err := mem.resize_bytes(last, 256, 16, allocator) + testing.expect_value(t, err, nil) + if testing.expect_value(t, len(s.leaked_allocations), 2) { + testing.expect(t, raw_data(s.leaked_allocations[0]) == raw_data(first), "resizing freed the wrong allocation") + testing.expect(t, raw_data(s.leaked_allocations[1]) == raw_data(grown)) + } +} + +@(test) +test_scratch_resize_nil :: proc(t: ^testing.T) { + s: mem.Scratch + mem.scratch_init(&s, 1024) + defer mem.scratch_destroy(&s) + + data, err := mem.scratch_resize_bytes(&s, nil, 64) + testing.expect_value(t, err, nil) + testing.expect_value(t, len(data), 64) + + start := uintptr(raw_data(s.data)) + ptr := uintptr(raw_data(data)) + testing.expect(t, start <= ptr && ptr+64 <= start+uintptr(len(s.data)), "resizing nil did not allocate from the buffer") +} + +@(test) +test_arena_resize :: proc(t: ^testing.T) { + buf: [4096]byte + mem.set(&buf, 0xCC, len(buf)) + + arena: mem.Arena + mem.arena_init(&arena, buf[:]) + expect_resize_keeps_bytes(t, mem.arena_allocator(&arena)) + expect_resize_realigns(t, mem.arena_allocator(&arena)) +} + +@(test) +test_dynamic_arena_resize :: proc(t: ^testing.T) { + arena: mem.Dynamic_Arena + mem.dynamic_arena_init(&arena) + defer mem.dynamic_arena_destroy(&arena) + + expect_resize_keeps_bytes(t, mem.dynamic_arena_allocator(&arena)) + expect_resize_realigns(t, mem.dynamic_arena_allocator(&arena)) +} + +@(test) +test_rollback_resize :: proc(t: ^testing.T) { + buf: [4096]byte + mem.set(&buf, 0xCC, len(buf)) + + rb: mem.Rollback_Stack + mem.rollback_stack_init(&rb, buf[:]) + expect_resize_keeps_bytes(t, mem.rollback_stack_allocator(&rb)) + expect_resize_realigns(t, mem.rollback_stack_allocator(&rb)) +} + +@(test) +test_rollback_resize_nil :: proc(t: ^testing.T) { + buf: [1024]byte + rb: mem.Rollback_Stack + mem.rollback_stack_init(&rb, buf[:]) + + data, err := mem.rb_resize_bytes(&rb, nil, 64) + testing.expect_value(t, err, nil) + testing.expect_value(t, len(data), 64) + testing.expect(t, raw_data(data) != nil) +} + +@(test) +test_rollback_resize_to_end :: proc(t: ^testing.T) { + buf: [1024]byte + rb: mem.Rollback_Stack + mem.rollback_stack_init(&rb, buf[:]) + allocator := mem.rollback_stack_allocator(&rb) + + // Growing the last allocation to exactly the end of the buffer fits. + data, _ := mem.alloc_bytes(64, 8, allocator) + room := len(rb.head.buffer) - int(rb.head.offset) + len(data) + + grown, err := mem.resize_bytes(data, room, 8, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(grown) == raw_data(data), "the last allocation did not grow in place") + testing.expect_value(t, int(rb.head.offset), len(rb.head.buffer)) +} + +@(test) +test_buddy_resize :: proc(t: ^testing.T) { + buf, _ := mem.make_aligned([]byte, 8192, 64) + defer delete(buf) + mem.set(raw_data(buf), 0xCC, len(buf)) + + b: mem.Buddy_Allocator + mem.buddy_allocator_init(&b, buf, 16) + expect_resize_keeps_bytes(t, mem.buddy_allocator(&b)) +} + +@(test) +test_compat_resize :: proc(t: ^testing.T) { + buf: [4096]byte + mem.set(&buf, 0xCC, len(buf)) + + arena: mem.Arena + mem.arena_init(&arena, buf[:]) + c: mem.Compat_Allocator + mem.compat_allocator_init(&c, mem.arena_allocator(&arena)) + expect_resize_keeps_bytes(t, mem.compat_allocator(&c)) + expect_resize_realigns(t, mem.compat_allocator(&c)) +} + +@(test) +test_compat_resize_nil :: proc(t: ^testing.T) { + c: mem.Compat_Allocator + mem.compat_allocator_init(&c) + allocator := mem.compat_allocator(&c) + + // `mem.resize` deals with nil itself, so call the procedure directly. + data, err := allocator.procedure(allocator.data, .Resize, 64, 16, nil, 0) + testing.expect_value(t, err, nil) + testing.expect_value(t, len(data), 64) + testing.expect_value(t, count_byte(data, 0), 64) + mem.free_bytes(data, allocator) +} + +@(test) +test_tlsf_resize :: proc(t: ^testing.T) { + alloc: tlsf.Allocator + defer tlsf.destroy(&alloc) + if err := tlsf.init_from_allocator(&alloc, context.allocator, 1 << 16); err != .None { + testing.fail_now(t, "TLSF init error") + } + + expect_resize_keeps_bytes(t, tlsf.allocator(&alloc)) + expect_resize_realigns(t, tlsf.allocator(&alloc)) +} + +@(require_results) +init_virtual_arena :: proc(arena: ^virtual.Arena, kind: virtual.Arena_Kind, buffer: []byte) -> mem.Allocator_Error { + switch kind { + case .Growing: return virtual.arena_init_growing(arena) + case .Static: return virtual.arena_init_static(arena) + case .Buffer: return virtual.arena_init_buffer(arena, buffer) + } + return nil +} + +@(test) +test_virtual_arena_resize :: proc(t: ^testing.T) { + for kind in virtual.Arena_Kind { + buf: [8192]byte + arena: virtual.Arena + testing.expect_value(t, init_virtual_arena(&arena, kind, buf[:]), nil) + defer virtual.arena_destroy(&arena) + + expect_resize_keeps_bytes(t, virtual.arena_allocator(&arena)) + expect_resize_realigns(t, virtual.arena_allocator(&arena)) + } +} + +@(test) +test_scratch_resize_to_end :: proc(t: ^testing.T) { + s: mem.Scratch + mem.scratch_init(&s, 1024) + defer mem.scratch_destroy(&s) + allocator := mem.scratch_allocator(&s) + + data, _ := mem.alloc_bytes(64, 16, allocator) + room := len(s.data) - int(uintptr(raw_data(data)) - uintptr(raw_data(s.data))) + + grown, err := mem.resize_bytes(data, room, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(grown) == raw_data(data), "the last allocation did not grow in place") +} + +@(test) +test_arena_resize_in_place :: proc(t: ^testing.T) { + buf: [1024]byte + mem.set(&buf, 0xCC, len(buf)) + arena: mem.Arena + mem.arena_init(&arena, buf[:]) + allocator := mem.arena_allocator(&arena) + + first, _ := mem.alloc_bytes(64, 16, allocator) + last, _ := mem.alloc_bytes(64, 16, allocator) + mem.set(raw_data(last), 0xAA, len(last)) + offset := arena.offset + + grown, err := mem.resize_bytes(last, 128, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place") + testing.expect_value(t, arena.offset, offset+64) + testing.expect_value(t, arena.peak_used, offset+64) + testing.expect_value(t, count_byte(grown[:64], 0xAA), 64) + testing.expect_value(t, count_byte(grown[64:], 0), 64) + + shrunk, shrunk_err := mem.resize_bytes(grown, 32, 16, allocator) + testing.expect_value(t, shrunk_err, nil) + testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place") + testing.expect_value(t, arena.offset, offset-32) + + // An earlier allocation shrinks in place but keeps its tail, and moves to grow. + offset = arena.offset + first_shrunk, _ := mem.resize_bytes(first, 16, 16, allocator) + testing.expect(t, raw_data(first_shrunk) == raw_data(first), "an earlier allocation did not shrink in place") + testing.expect_value(t, arena.offset, offset) + first_grown, _ := mem.resize_bytes(first_shrunk, 128, 16, allocator) + testing.expect(t, raw_data(first_grown) != raw_data(first), "an earlier allocation grew in place over a later one") + testing.expect_value(t, count_byte(shrunk, 0xAA), 32) + + // While a temporary memory region is active, a shrunk last allocation keeps its tail. + offset = arena.offset + tmp := mem.begin_arena_temp_memory(&arena) + _, _ = mem.resize_bytes(first_grown, 64, 16, allocator) + testing.expect_value(t, arena.offset, offset) + mem.end_arena_temp_memory(tmp) + testing.expect_value(t, arena.offset, offset) +} + +@(test) +test_small_stack_resize_in_place :: proc(t: ^testing.T) { + buf: [1024]byte + s: mem.Small_Stack + mem.small_stack_init(&s, buf[:]) + allocator := mem.small_stack_allocator(&s) + + first, _ := mem.alloc_bytes(64, 16, allocator) + last, _ := mem.alloc_bytes(64, 16, allocator) + mem.set(raw_data(last), 0xAA, len(last)) + offset := s.offset + + grown, err := mem.resize_bytes(last, 128, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place") + testing.expect_value(t, s.offset, offset+64) + testing.expect_value(t, s.peak_used, offset+64) + + shrunk, shrunk_err := mem.resize_bytes(grown, 32, 16, allocator) + testing.expect_value(t, shrunk_err, nil) + testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place") + testing.expect_value(t, s.offset, offset-32) + + // An earlier allocation shrinks in place, and moves to grow. + offset = s.offset + first_shrunk, _ := mem.resize_bytes(first, 16, 16, allocator) + testing.expect(t, raw_data(first_shrunk) == raw_data(first), "an earlier allocation did not shrink in place") + testing.expect_value(t, s.offset, offset) + first_grown, _ := mem.resize_bytes(first_shrunk, 128, 16, allocator) + testing.expect(t, raw_data(first_grown) != raw_data(first), "an earlier allocation grew in place over a later one") + testing.expect_value(t, count_byte(shrunk, 0xAA), 32) +} + +@(test) +test_dynamic_arena_resize_in_place :: proc(t: ^testing.T) { + arena: mem.Dynamic_Arena + mem.dynamic_arena_init(&arena) + defer mem.dynamic_arena_destroy(&arena) + allocator := mem.dynamic_arena_allocator(&arena) + + last, _ := mem.alloc_bytes(64, 16, allocator) + mem.set(raw_data(last), 0xAA, len(last)) + bytes_left := arena.bytes_left + + grown, err := mem.resize_bytes(last, 128, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place") + testing.expect_value(t, arena.bytes_left, bytes_left-64) + testing.expect_value(t, count_byte(grown[:64], 0xAA), 64) + testing.expect_value(t, count_byte(grown[64:], 0), 64) + + shrunk, shrunk_err := mem.resize_bytes(grown, 32, 16, allocator) + testing.expect_value(t, shrunk_err, nil) + testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place") + testing.expect_value(t, arena.bytes_left, bytes_left+32) + + // An allocation followed by another is not the last one, even when a + // stricter alignment would round its size up to reach past the later one. + first: []byte + for { + first, _ = mem.alloc_bytes(16, 16, allocator) + if mem.is_aligned(raw_data(first), 64) { + break + } + } + later, _ := mem.alloc_bytes(48, 16, allocator) + mem.set(raw_data(later), 0xBB, len(later)) + + first_grown, _ := mem.resize_bytes(first, 64, 64, allocator) + testing.expect(t, raw_data(first_grown) != raw_data(first), "an earlier allocation grew in place over a later one") + testing.expect_value(t, count_byte(later, 0xBB), 48) +} + +@(test) +test_virtual_arena_resize_in_place :: proc(t: ^testing.T) { + for kind in virtual.Arena_Kind { + buf: [16384]byte + arena: virtual.Arena + testing.expect_value(t, init_virtual_arena(&arena, kind, buf[:]), nil) + defer virtual.arena_destroy(&arena) + allocator := virtual.arena_allocator(&arena) + + data, _ := mem.alloc_bytes(4096, 16, allocator) + mem.set(raw_data(data), 0xAA, len(data)) + used := arena.curr_block.used + total := arena.total_used + + // Shrinking the last allocation gives its tail back, zeroed. + shrunk, err := mem.resize_bytes(data, 1000, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(shrunk) == raw_data(data), "the last allocation did not shrink in place") + testing.expect_value(t, count_byte(shrunk, 0xAA), 1000) + testing.expect_value(t, arena.curr_block.used, used-3096) + testing.expect_value(t, arena.total_used, total-3096) + + next, _ := mem.alloc_bytes(3000, 1, allocator) + testing.expect(t, raw_data(next) == raw_data(shrunk[1000:]), "the tail was not given back") + testing.expect_value(t, count_byte(next, 0), 3000) + + // An earlier allocation shrinks in place, keeping its tail. + used = arena.curr_block.used + _, _ = mem.resize_bytes(shrunk, 500, 16, allocator) + testing.expect_value(t, arena.curr_block.used, used) + } +} + +@(test) +test_virtual_arena_shrink_in_temp :: proc(t: ^testing.T) { + for kind in virtual.Arena_Kind { + buf: [16384]byte + arena: virtual.Arena + testing.expect_value(t, init_virtual_arena(&arena, kind, buf[:]), nil) + defer virtual.arena_destroy(&arena) + allocator := virtual.arena_allocator(&arena) + + data, _ := mem.alloc_bytes(4096, 16, allocator) + used := arena.curr_block.used + + // The temporary region saved the end of the allocation, so it keeps its tail. + temp := virtual.arena_temp_begin(&arena) + shrunk, err := mem.resize_bytes(data, 1000, 16, allocator) + testing.expect_value(t, err, nil) + testing.expect(t, raw_data(shrunk) == raw_data(data), "the last allocation did not shrink in place") + testing.expect_value(t, arena.curr_block.used, used) + virtual.arena_temp_end(temp) + testing.expect_value(t, arena.curr_block.used, used) + } +}