Merge pull request #7962 from odin-lang/bill/allocator-fixes

Fix and improve resize in `core:mem` allocators

core/mem/allocators.odin

@@ -289,6 +289,145 @@ arena_free_all :: proc(a: ^Arena) {
 	// sanitizer.address_poison(a.data)
 }
 
+/*
+Resize an allocation owned by an arena.
+
+This procedure resizes a memory region defined by its location `old_memory` and its size `old_size` to have a size `size` and alignment `alignment`.
+The newly allocated memory, if any, is zero-initialized.
+
+If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size,
+aligned on a boundary specified by `alignment`.
+
+The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks,
+unless a temporary memory region is active. Any other allocation shrinks in place, and moves to grow.
+
+This procedure returns the pointer to the resized memory region.
+*/
+@(require_results)
+arena_resize :: proc(
+	a:          ^Arena,
+	old_memory: rawptr,
+	old_size:   int,
+	size:       int,
+	alignment := DEFAULT_ALIGNMENT,
+	loc       := #caller_location,
+) -> (rawptr, Allocator_Error) {
+	bytes, err := arena_resize_bytes(a, byte_slice(old_memory, old_size), size, alignment, loc)
+	return raw_data(bytes), err
+}
+
+/*
+Resize an allocation owned by an arena.
+
+This procedure resizes a memory region specified by `old_data` to have a size `size` and alignment `alignment`.
+The newly allocated memory, if any, is zero-initialized.
+
+If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size,
+aligned on a boundary specified by `alignment`.
+
+The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks, unless a temporary memory region is active.
+Any other allocation shrinks in place and moves to grow.
+
+This procedure returns the slice of the resized memory region.
+*/
+@(require_results)
+arena_resize_bytes :: proc(
+	a:        ^Arena,
+	old_data: []byte,
+	size:     int,
+	alignment := DEFAULT_ALIGNMENT,
+	loc       := #caller_location,
+) -> ([]byte, Allocator_Error) {
+	bytes, err := arena_resize_bytes_non_zeroed(a, old_data, size, alignment, loc)
+	if bytes != nil && size > len(old_data) {
+		zero_slice(bytes[len(old_data):])
+	}
+	return bytes, err
+}
+
+/*
+Resize an allocation owned by an arena, without zero-initialization.
+
+This procedure resizes a memory region defined by its location `old_memory` and its size `old_size` to have a size `size` and alignment `alignment`.
+The newly allocated memory, if any, is not explicitly zero-initialized.
+
+If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size,
+aligned on a boundary specified by `alignment`.
+
+The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks,
+unless a temporary memory region is active. Any other allocation shrinks in place, and moves to grow.
+
+This procedure returns the pointer to the resized memory region.
+*/
+@(require_results)
+arena_resize_non_zeroed :: proc(
+	a:          ^Arena,
+	old_memory: rawptr,
+	old_size:   int,
+	size:       int,
+	alignment := DEFAULT_ALIGNMENT,
+	loc       := #caller_location,
+) -> (rawptr, Allocator_Error) {
+	bytes, err := arena_resize_bytes_non_zeroed(a, byte_slice(old_memory, old_size), size, alignment, loc)
+	return raw_data(bytes), err
+}
+
+/*
+Resize an allocation owned by an arena, without zero-initialization.
+
+This procedure resizes a memory region specified by `old_data` to have a size `size` and alignment `alignment`.
+The newly allocated memory, if any, is not explicitly zero-initialized.
+
+If `old_memory` is `nil`, this procedure acts just like `arena_alloc()`, allocating a memory region `size` bytes in size,
+aligned on a boundary specified by `alignment`.
+
+The last allocation is resized in place if it fits, and gives its tail back to the arena when it shrinks,
+unless a temporary memory region is active. Any other allocation shrinks in place, and moves to grow.
+
+This procedure returns the slice of the resized memory region.
+*/
+@(require_results)
+arena_resize_bytes_non_zeroed :: proc(
+	a:        ^Arena,
+	old_data: []byte,
+	size:     int,
+	alignment := DEFAULT_ALIGNMENT,
+	loc       := #caller_location,
+) -> ([]byte, Allocator_Error) {
+	old_memory := raw_data(old_data)
+	old_size   := len(old_data)
+	if old_memory == nil {
+		return arena_alloc_bytes_non_zeroed(a, size, alignment, loc)
+	}
+	if is_aligned(old_memory, alignment) {
+		begin   := uintptr(raw_data(a.data))
+		old_ptr := uintptr(old_memory)
+		is_last := begin <= old_ptr && old_ptr+uintptr(old_size) == begin+uintptr(a.offset)
+		switch {
+		case is_last && int(old_ptr-begin)+size <= len(a.data):
+			// NOTE(bill): A temporary memory region may have saved an offset inside the tail; therefore keep the tail while one is active.
+			if size > old_size || a.temp_count == 0 {
+				// sanitizer.address_poison(old_data[size:])
+				a.offset    = int(old_ptr-begin) + size
+				a.peak_used = max(a.peak_used, a.offset)
+			}
+			result := byte_slice(old_memory, size)
+			// sanitizer.address_unpoison(result)
+			return result, nil
+
+		case size <= old_size:
+			// sanitizer.address_poison(old_data[size:])
+			return byte_slice(old_memory, size), nil
+		}
+	}
+	data, err := arena_alloc_bytes_non_zeroed(a, size, alignment, loc)
+	if err == nil {
+		runtime.copy(data, old_data)
+		// sanitizer.address_poison(old_data)
+	}
+	return data, err
+}
+
 arena_allocator_proc :: proc(
 	allocator_data: rawptr,
 	mode:           Allocator_Mode,
@@ -309,9 +448,9 @@ arena_allocator_proc :: proc(
 	case .Free_All:
 		arena_free_all(arena)
 	case .Resize:
-		return default_resize_bytes_align(byte_slice(old_memory, old_size), size, alignment, arena_allocator(arena), loc)
+		return arena_resize_bytes(arena, byte_slice(old_memory, old_size), size, alignment, loc)
 	case .Resize_Non_Zeroed:
-		return default_resize_bytes_align_non_zeroed(byte_slice(old_memory, old_size), size, alignment, arena_allocator(arena), loc)
+		return arena_resize_bytes_non_zeroed(arena, byte_slice(old_memory, old_size), size, alignment, loc)
 	case .Query_Features:
 		set := (^Allocator_Mode_Set)(old_memory)
 		if set != nil {
@@ -607,8 +746,7 @@ scratch_free :: proc(s: ^Scratch, ptr: rawptr, loc := #caller_location) -> Alloc
 	}
 	if len(s.leaked_allocations) != 0 {
 		for data, i in s.leaked_allocations {
-			ptr := raw_data(data)
-			if ptr == ptr {
+			if raw_data(data) == ptr {
 				free_bytes(data, s.backup_allocator, loc)
 				ordered_remove(&s.leaked_allocations, i, loc)
 				return nil
@@ -686,7 +824,7 @@ scratch_resize_bytes :: proc(
 ) -> ([]byte, Allocator_Error) {
 	bytes, err := scratch_resize_bytes_non_zeroed(s, old_data, size, alignment, loc)
 	if bytes != nil && size > len(old_data) {
-		zero_slice(bytes[size:])
+		zero_slice(bytes[len(old_data):])
 	}
 	return bytes, err
 }
@@ -753,6 +891,9 @@ scratch_resize_bytes_non_zeroed :: proc(
 		}
 		scratch_init(s, DEFAULT_BACKING_SIZE)
 	}
+	if old_memory == nil {
+		return scratch_alloc_bytes_non_zeroed(s, size, alignment, loc)
+	}
 	begin   := uintptr(raw_data(s.data))
 	end     := begin + uintptr(len(s.data))
 	old_ptr := uintptr(old_memory)
@@ -761,7 +902,7 @@ scratch_resize_bytes_non_zeroed :: proc(
 	//
 	// Also, the alignments must match, otherwise we must re-allocate to
 	// guarantee the user's request.
-	if s.prev_allocation == old_memory && is_aligned(old_memory, alignment) && old_ptr+uintptr(size) < end {
+	if s.prev_allocation == old_memory && is_aligned(old_memory, alignment) && old_ptr+uintptr(size) <= end {
 		// ensure_not_poisoned(old_data)
 		// sanitizer.address_poison(old_memory)
 		s.curr_offset = int(old_ptr-begin)+size
@@ -1172,7 +1313,8 @@ stack_resize_bytes_non_zeroed :: proc(
 	}
 	header := (^Stack_Allocation_Header)(curr_addr - size_of(Stack_Allocation_Header))
 	old_offset := int(curr_addr - uintptr(header.padding) - uintptr(raw_data(s.data)))
-	if old_offset != header.prev_offset {
+	if old_offset != s.prev_offset {
+		// Not the last allocation, so it cannot be resized in place.
 		data, err := stack_alloc_bytes_non_zeroed(s, size, alignment, loc)
 		if err == nil {
 			runtime.copy(data, byte_slice(old_memory, old_size))
@@ -1183,10 +1325,12 @@ stack_resize_bytes_non_zeroed :: proc(
 	old_memory_size := uintptr(s.curr_offset) - (curr_addr - start)
 	assert(old_memory_size == uintptr(old_size))
 	diff := size - old_size
+	if s.curr_offset + diff > len(s.data) {
+		return nil, .Out_Of_Memory
+	}
 	s.curr_offset += diff // works for smaller sizes too
-	if diff > 0 {
-		zero(rawptr(curr_addr + uintptr(diff)), diff)
-	} else {
+	s.peak_used = max(s.peak_used, s.curr_offset)
+	if diff < 0 {
 		// sanitizer.address_poison(old_data[size:])
 	}
 	result := byte_slice(old_memory, size)
@@ -1592,6 +1736,20 @@ small_stack_resize_bytes_non_zeroed :: proc(
 		// sanitizer.address_unpoison(result)
 		return result, nil
 	}
+	is_last := curr_addr+uintptr(old_size) == start+uintptr(s.offset)
+	switch {
+	case is_last && int(curr_addr-start)+size <= len(s.data):
+		// sanitizer.address_poison(old_data[size:])
+		s.offset    = int(curr_addr-start) + size
+		s.peak_used = max(s.peak_used, s.offset)
+		result := byte_slice(old_memory, size)
+		// sanitizer.address_unpoison(result)
+		return result, nil
+
+	case size < old_size:
+		// sanitizer.address_poison(old_data[size:])
+		return byte_slice(old_memory, size), nil
+	}
 	data, err := small_stack_alloc_bytes_non_zeroed(s, size, alignment, loc)
 	if err == nil {
 		runtime.copy(data, byte_slice(old_memory, old_size))
@@ -2002,12 +2160,34 @@ dynamic_arena_resize_bytes_non_zeroed :: proc(
 	}
 	old_memory := raw_data(old_data)
 	old_size := len(old_data)
-	if old_size >= size {
-		// sanitizer.address_poison(old_data[size:])
-		return byte_slice(old_memory, size), nil
+	if is_aligned(old_memory, alignment) {
+		// No information is kept about allocations in this allocator, but the
+		// last one in the current block ends less than `minimum_alignment`
+		// bytes before `current_pos`, as any later one takes at least that
+		// many bytes. It can be resized in place.
+		old_end   := uintptr(old_memory) + uintptr(old_size)
+		curr_pos  := uintptr(a.current_pos)
+		block_end := curr_pos + uintptr(a.bytes_left)
+		if old_size > 0                                    &&
+		   uintptr(a.current_block) <= uintptr(old_memory) &&
+		   old_end <= curr_pos                             &&
+		   curr_pos-old_end < uintptr(a.minimum_alignment) {
+			new_pos := uintptr(old_memory) + uintptr(align_formula(size, max(a.minimum_alignment, alignment)))
+			if new_pos <= block_end {
+				// sanitizer.address_poison(old_data[size:])
+				a.current_pos = rawptr(new_pos)
+				a.bytes_left  = int(block_end - new_pos)
+				result := byte_slice(old_memory, size)
+				// sanitizer.address_unpoison(result)
+				return result, nil
+			}
+		}
+		if old_size >= size {
+			// sanitizer.address_poison(old_data[size:])
+			return byte_slice(old_memory, size), nil
+		}
 	}
-	// No information is kept about allocations in this allocator, thus we
-	// cannot truly resize anything and must reallocate.
+	// Any other allocation cannot truly be resized, and must be reallocated.
 	data, err := dynamic_arena_alloc_bytes_non_zeroed(a, size, alignment, loc)
 	if err == nil {
 		runtime.copy(data, byte_slice(old_memory, old_size))
@@ -2491,6 +2671,11 @@ compat_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode,
 		return rra.parent.procedure(rra.parent.data, mode, orig_size, header.alignment, orig_ptr, orig_size, location)
 
 	case .Resize, .Resize_Non_Zeroed:
+		if old_memory == nil {
+			alloc_mode := Allocator_Mode.Alloc if mode == .Resize else .Alloc_Non_Zeroed
+			return compat_allocator_proc(allocator_data, alloc_mode, size, alignment, nil, 0, location)
+		}
+
 		header    := get_unpoisoned_header(old_memory)
 		orig_a    := max(header.alignment, size_of(Header))
 		orig_ptr  := rawptr(uintptr(old_memory)-uintptr(orig_a))
@@ -2503,6 +2688,10 @@ compat_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode,
 		assert(size >= 0, "overflow")
 
 		allocation := rra.parent.procedure(rra.parent.data, mode, req_size, new_alignment, orig_ptr, orig_size, location) or_return
+		if a != orig_a {
+			// The header grew with the alignment, so the data must move up after it.
+			runtime.copy(allocation[a:], allocation[orig_a:][:min(header.size, size)])
+		}
 		#no_bounds_check data = allocation[a:]
 
 		([^]Header)(raw_data(data))[-1] = {

core/mem/rollback_stack_allocator.odin

@@ -326,12 +326,15 @@ rb_resize_bytes_non_zeroed :: proc(
 	assert(size >= 0, "Size must be positive or zero.", loc)
 	assert(old_size >= 0, "Old size must be positive or zero.", loc)
 	assert(is_power_of_two(cast(uintptr)alignment), "Alignment must be a power of two.", loc)
-	if ptr != nil {
+	if ptr == nil {
+		return rb_alloc_bytes_non_zeroed(stack, size, alignment, loc)
+	}
+	if is_aligned(ptr, alignment) {
 		if block, _, ok := rb_find_last_alloc(stack, ptr); ok {
 			// `block.offset` should never underflow because it is contingent
 			// on `old_size` in the first place, assuming sane arguments.
 			assert(block.offset >= cast(uintptr)old_size, "Rollback Stack Allocator received invalid `old_size`.")
-			if block.offset + cast(uintptr)size - cast(uintptr)old_size < cast(uintptr)len(block.buffer) {
+			if block.offset + cast(uintptr)size - cast(uintptr)old_size <= cast(uintptr)len(block.buffer) {
 				// Prevent singleton allocations from fragmenting by forbidding
 				// them to shrink, removing the possibility of overflow bugs.
 				if len(block.buffer) <= stack.block_size {

core/mem/tlsf/tlsf_internal.odin

@@ -306,7 +306,9 @@ resize :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size: uint, align
 
 	min_size := min(curr_size, new_size, old_size)
 
-	if adjust > curr_size && (!block_is_free(next) || adjust > combined) {
+	// NOTE(bill): A stricter alignment than the block has cannot be met in place.
+	misaligned := uintptr(ptr) % uintptr(max(ALIGN_SIZE, alignment)) != 0
+	if misaligned || adjust > curr_size && (!block_is_free(next) || adjust > combined) {
 		res = alloc_bytes(control, new_size, alignment) or_return
 		if res != nil {
 			copy(res, ([^]byte)(ptr)[:min_size])
@@ -351,7 +353,9 @@ resize_non_zeroed :: proc(control: ^Allocator, ptr: rawptr, old_size, new_size:
 
 	min_size := min(curr_size, new_size, old_size)
 
-	if adjust > curr_size && (!block_is_free(next) || adjust > combined) {
+	// NOTE(bill): A stricter alignment than the block has cannot be met in place.
+	misaligned := uintptr(ptr) % uintptr(max(ALIGN_SIZE, alignment)) != 0
+	if misaligned || adjust > curr_size && (!block_is_free(next) || adjust > combined) {
 		res = alloc_bytes_non_zeroed(control, new_size, alignment) or_return
 		if res != nil {
 			copy(res, ([^]byte)(ptr)[:min_size])

core/mem/virtual/arena.odin

@@ -347,7 +347,7 @@ arena_allocator_proc :: proc(allocator_data: rawptr, mode: mem.Allocator_Mode,
 		switch {
 		case old_data == nil:
 			return arena_alloc(arena, size, alignment, location)
-		case size == old_size:
+		case size == old_size && uintptr(old_data) & uintptr(alignment-1) == 0:
 			// return old memory
 			data = old_data[:size]
 			return
@@ -359,27 +359,41 @@ arena_allocator_proc :: proc(allocator_data: rawptr, mode: mem.Allocator_Mode,
 		sync.mutex_guard(&arena.mutex)
 
 		if uintptr(old_data) & uintptr(alignment-1) == 0 {
-			if size < old_size {
-				// shrink data in-place
-				data = old_data[:size]
-				// sanitizer.address_poison(old_data[size:old_size])
-				return
-			}
-
 			if block := arena.curr_block; block != nil {
 				start := uint(uintptr(old_memory)) - uint(uintptr(block.base))
 				old_end := start + old_size
 				new_end := start + size
 				if start < old_end && old_end == block.used && new_end <= block.reserved {
-					// grow data in-place, adjusting next allocation
-					prev_used := block.used
-					_ = alloc_from_memory_block(block, new_end - old_end, 1, default_commit_size=arena.default_commit_size) or_return
-					arena.total_used += block.used - prev_used
+					switch {
+					case size > old_size:
+						// Grow data in-place by adjusting the next allocation
+						prev_used := block.used
+						_ = alloc_from_memory_block(block, new_end - old_end, 1, default_commit_size=arena.default_commit_size) or_return
+						arena.total_used += block.used - prev_used
+
+					case arena.temp_count == 0:
+						// Shrink data in-place by giving the tail back to the block
+						//
+						// NOTE(bill): This is not done while an `Arena_Temp` is active.
+						// It may have saved a position inside the tail.
+						// The memory past `used` is kept zeroed because `.Alloc` relies on it.
+						mem.zero_slice(block.base[new_end:old_end])
+						// sanitizer.address_poison(block.base[new_end:old_end])
+						block.used = new_end
+						arena.total_used -= old_size - size
+					}
 					data = block.base[start:new_end]
 					// sanitizer.address_unpoison(data)
 					return
 				}
 			}
+
+			if size < old_size {
+				// Allow for shrinking data in-place
+				data = old_data[:size]
+				// sanitizer.address_poison(old_data[size:old_size])
+				return
+			}
 		}
 
 		new_memory := arena_alloc_unguarded(arena, size, alignment, location) or_return

tests/core/mem/test_mem_resize.odin

@@ -0,0 +1,543 @@
+package test_core_mem
+
+import "core:mem"
+import "core:mem/tlsf"
+import "core:mem/virtual"
+import "core:testing"
+
+@(require_results)
+count_byte :: proc(data: []byte, value: byte) -> (n: int) {
+	for b in data {
+		if b == value {
+			n += 1
+		}
+	}
+	return
+}
+
+/*
+Grow, grow without zeroing, then shrink one allocation, checking that the old
+bytes survive each step and that the zeroing resize clears the new tail.
+
+Fill the backing memory with non-zero bytes first where possible, so that a
+tail which is not zeroed shows up.
+*/
+expect_resize_keeps_bytes :: proc(t: ^testing.T, allocator: mem.Allocator, loc := #caller_location) {
+	data, err := mem.alloc_bytes(100, 16, allocator)
+	if !testing.expect_value(t, err, nil, loc) {
+		return
+	}
+	data = data[:100] // TLSF hands back the whole block
+	mem.set(raw_data(data), 0xAA, len(data))
+
+	data, err = mem.resize_bytes(data, 150, 16, allocator)
+	if !testing.expect_value(t, err, nil, loc) {
+		return
+	}
+	testing.expect_value(t, count_byte(data[:100], 0xAA), 100, loc)
+	testing.expect_value(t, count_byte(data[100:], 0),     50,  loc)
+
+	data, err = mem.resize_bytes_non_zeroed(data, 200, 16, allocator)
+	if !testing.expect_value(t, err, nil, loc) {
+		return
+	}
+	testing.expect_value(t, count_byte(data[:100],    0xAA), 100, loc)
+	testing.expect_value(t, count_byte(data[100:150], 0),    50,  loc)
+
+	data, err = mem.resize_bytes(data, 50, 16, allocator)
+	if !testing.expect_value(t, err, nil, loc) {
+		return
+	}
+	testing.expect_value(t, count_byte(data, 0xAA), 50, loc)
+	_ = mem.free_bytes(data, allocator)
+}
+
+// Allocate 48 bytes off a 64-byte boundary, holding their own indices.
+@(require_results)
+alloc_off_64 :: proc(t: ^testing.T, allocator: mem.Allocator, loc := #caller_location) -> (data: []byte, ok: bool) {
+	for i in 0..<8 {
+		data, _ = mem.alloc_bytes(48, 16, allocator)
+		if data != nil && !mem.is_aligned(raw_data(data), 64) {
+			data = data[:48]
+			for &b, j in data {
+				b = byte(j)
+			}
+			return data, true
+		}
+		// Shift the next allocation along, by a different amount each time.
+		_, _ = mem.alloc_bytes(16*(i+1), 16, allocator)
+	}
+	testing.expect(t, false, "found no allocation off a 64-byte boundary", loc=loc)
+	return nil, false
+}
+
+// Resizing to a stricter alignment than the allocation has must move it, keeping its bytes.
+expect_resize_realigns :: proc(t: ^testing.T, allocator: mem.Allocator, loc := #caller_location) {
+	for size in ([]int{48, 32, 96}) {
+		data := alloc_off_64(t, allocator, loc) or_break
+
+		resized, err := mem.resize_bytes(data, size, 64, allocator)
+		if !testing.expect_value(t, err, nil, loc) {
+			continue
+		}
+		testing.expectf(t, mem.is_aligned(raw_data(resized), 64), "resizing to %d bytes left them at %p", size, raw_data(resized), loc=loc)
+
+		kept := 0
+		for b, i in resized[:min(size, 48)] {
+			if b == byte(i) {
+				kept += 1
+			}
+		}
+		testing.expectf(t, kept == min(size, 48), "resizing to %d bytes kept %d of the old bytes", size, kept, loc=loc)
+		_ = mem.free_bytes(resized, allocator)
+	}
+}
+
+@(test)
+test_stack_resize :: proc(t: ^testing.T) {
+	buf: [4096]byte
+	mem.set(&buf, 0xCC, len(buf))
+
+	s: mem.Stack
+	mem.stack_init(&s, buf[:])
+	expect_resize_keeps_bytes(t, mem.stack_allocator(&s))
+	expect_resize_realigns(t, mem.stack_allocator(&s))
+}
+
+@(test)
+test_stack_resize_out_of_memory :: proc(t: ^testing.T) {
+	// The rest of `buf` takes any write past the end of the stack.
+	buf: [8192]byte
+	s: mem.Stack
+	mem.stack_init(&s, buf[:1024])
+	allocator := mem.stack_allocator(&s)
+
+	data, _ := mem.alloc_bytes(512, 16, allocator)
+	offset := s.curr_offset
+
+	grown, err := mem.resize_bytes(data, 4096, 16, allocator)
+	testing.expect_value(t, err, mem.Allocator_Error.Out_Of_Memory)
+	testing.expect(t, grown == nil)
+	testing.expect_value(t, s.curr_offset, offset)
+}
+
+@(test)
+test_stack_resize_last_allocation :: proc(t: ^testing.T) {
+	buf: [1024]byte
+	s: mem.Stack
+	mem.stack_init(&s, buf[:])
+	allocator := mem.stack_allocator(&s)
+
+	_, _ = mem.alloc_bytes(100, 16, allocator)
+	last, _ := mem.alloc_bytes(100, 16, allocator)
+	offset := s.curr_offset
+
+	grown, err := mem.resize_bytes(last, 150, 16, allocator)
+	testing.expect_value(t, err, nil)
+	testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place")
+	testing.expect_value(t, s.curr_offset, offset+50)
+	testing.expect_value(t, s.peak_used,   offset+50)
+
+	shrunk, shrunk_err := mem.resize_bytes(grown, 20, 16, allocator)
+	testing.expect_value(t, shrunk_err, nil)
+	testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place")
+	testing.expect_value(t, s.curr_offset, offset-80)
+}
+
+@(test)
+test_stack_resize_earlier_allocation :: proc(t: ^testing.T) {
+	buf: [1024]byte
+	s: mem.Stack
+	mem.stack_init(&s, buf[:])
+	allocator := mem.stack_allocator(&s)
+
+	first, _ := mem.alloc_bytes(100, 16, allocator)
+	last,  _ := mem.alloc_bytes(100, 16, allocator)
+	mem.set(raw_data(first), 0xAA, len(first))
+	mem.set(raw_data(last),  0xBB, len(last))
+
+	grown, err := mem.resize_bytes(first, 150, 16, allocator)
+	testing.expect_value(t, err, nil)
+	testing.expect(t, raw_data(grown) != raw_data(first), "an earlier allocation grew in place over a later one")
+	testing.expect_value(t, count_byte(grown[:100], 0xAA), 100)
+	testing.expect_value(t, count_byte(grown[100:], 0),    50)
+	testing.expect_value(t, count_byte(last,        0xBB), 100)
+}
+
+@(test)
+test_small_stack_resize :: proc(t: ^testing.T) {
+	buf: [4096]byte
+	mem.set(&buf, 0xCC, len(buf))
+
+	s: mem.Small_Stack
+	mem.small_stack_init(&s, buf[:])
+	expect_resize_keeps_bytes(t, mem.small_stack_allocator(&s))
+	expect_resize_realigns(t, mem.small_stack_allocator(&s))
+}
+
+@(test)
+test_scratch_resize :: proc(t: ^testing.T) {
+	s: mem.Scratch
+	mem.scratch_init(&s, 4096)
+	defer mem.scratch_destroy(&s)
+	mem.set(raw_data(s.data), 0xCC, len(s.data))
+
+	expect_resize_keeps_bytes(t, mem.scratch_allocator(&s))
+	expect_resize_realigns(t, mem.scratch_allocator(&s))
+}
+
+@(test)
+test_scratch_resize_leaked_allocation :: proc(t: ^testing.T) {
+	s: mem.Scratch
+	mem.scratch_init(&s, 64)
+	defer mem.scratch_destroy(&s)
+	allocator := mem.scratch_allocator(&s)
+
+	// Both are too big for the buffer, so they come from the backup allocator.
+	first, _ := mem.alloc_bytes(128, 16, allocator)
+	last,  _ := mem.alloc_bytes(128, 16, allocator)
+
+	grown, err := mem.resize_bytes(last, 256, 16, allocator)
+	testing.expect_value(t, err, nil)
+	if testing.expect_value(t, len(s.leaked_allocations), 2) {
+		testing.expect(t, raw_data(s.leaked_allocations[0]) == raw_data(first), "resizing freed the wrong allocation")
+		testing.expect(t, raw_data(s.leaked_allocations[1]) == raw_data(grown))
+	}
+}
+
+@(test)
+test_scratch_resize_nil :: proc(t: ^testing.T) {
+	s: mem.Scratch
+	mem.scratch_init(&s, 1024)
+	defer mem.scratch_destroy(&s)
+
+	data, err := mem.scratch_resize_bytes(&s, nil, 64)
+	testing.expect_value(t, err, nil)
+	testing.expect_value(t, len(data), 64)
+
+	start := uintptr(raw_data(s.data))
+	ptr   := uintptr(raw_data(data))
+	testing.expect(t, start <= ptr && ptr+64 <= start+uintptr(len(s.data)), "resizing nil did not allocate from the buffer")
+}
+
+@(test)
+test_arena_resize :: proc(t: ^testing.T) {
+	buf: [4096]byte
+	mem.set(&buf, 0xCC, len(buf))
+
+	arena: mem.Arena
+	mem.arena_init(&arena, buf[:])
+	expect_resize_keeps_bytes(t, mem.arena_allocator(&arena))
+	expect_resize_realigns(t, mem.arena_allocator(&arena))
+}
+
+@(test)
+test_dynamic_arena_resize :: proc(t: ^testing.T) {
+	arena: mem.Dynamic_Arena
+	mem.dynamic_arena_init(&arena)
+	defer mem.dynamic_arena_destroy(&arena)
+
+	expect_resize_keeps_bytes(t, mem.dynamic_arena_allocator(&arena))
+	expect_resize_realigns(t, mem.dynamic_arena_allocator(&arena))
+}
+
+@(test)
+test_rollback_resize :: proc(t: ^testing.T) {
+	buf: [4096]byte
+	mem.set(&buf, 0xCC, len(buf))
+
+	rb: mem.Rollback_Stack
+	mem.rollback_stack_init(&rb, buf[:])
+	expect_resize_keeps_bytes(t, mem.rollback_stack_allocator(&rb))
+	expect_resize_realigns(t, mem.rollback_stack_allocator(&rb))
+}
+
+@(test)
+test_rollback_resize_nil :: proc(t: ^testing.T) {
+	buf: [1024]byte
+	rb: mem.Rollback_Stack
+	mem.rollback_stack_init(&rb, buf[:])
+
+	data, err := mem.rb_resize_bytes(&rb, nil, 64)
+	testing.expect_value(t, err, nil)
+	testing.expect_value(t, len(data), 64)
+	testing.expect(t, raw_data(data) != nil)
+}
+
+@(test)
+test_rollback_resize_to_end :: proc(t: ^testing.T) {
+	buf: [1024]byte
+	rb: mem.Rollback_Stack
+	mem.rollback_stack_init(&rb, buf[:])
+	allocator := mem.rollback_stack_allocator(&rb)
+
+	// Growing the last allocation to exactly the end of the buffer fits.
+	data, _ := mem.alloc_bytes(64, 8, allocator)
+	room := len(rb.head.buffer) - int(rb.head.offset) + len(data)
+
+	grown, err := mem.resize_bytes(data, room, 8, allocator)
+	testing.expect_value(t, err, nil)
+	testing.expect(t, raw_data(grown) == raw_data(data), "the last allocation did not grow in place")
+	testing.expect_value(t, int(rb.head.offset), len(rb.head.buffer))
+}
+
+@(test)
+test_buddy_resize :: proc(t: ^testing.T) {
+	buf, _ := mem.make_aligned([]byte, 8192, 64)
+	defer delete(buf)
+	mem.set(raw_data(buf), 0xCC, len(buf))
+
+	b: mem.Buddy_Allocator
+	mem.buddy_allocator_init(&b, buf, 16)
+	expect_resize_keeps_bytes(t, mem.buddy_allocator(&b))
+}
+
+@(test)
+test_compat_resize :: proc(t: ^testing.T) {
+	buf: [4096]byte
+	mem.set(&buf, 0xCC, len(buf))
+
+	arena: mem.Arena
+	mem.arena_init(&arena, buf[:])
+	c: mem.Compat_Allocator
+	mem.compat_allocator_init(&c, mem.arena_allocator(&arena))
+	expect_resize_keeps_bytes(t, mem.compat_allocator(&c))
+	expect_resize_realigns(t, mem.compat_allocator(&c))
+}
+
+@(test)
+test_compat_resize_nil :: proc(t: ^testing.T) {
+	c: mem.Compat_Allocator
+	mem.compat_allocator_init(&c)
+	allocator := mem.compat_allocator(&c)
+
+	// `mem.resize` deals with nil itself, so call the procedure directly.
+	data, err := allocator.procedure(allocator.data, .Resize, 64, 16, nil, 0)
+	testing.expect_value(t, err, nil)
+	testing.expect_value(t, len(data), 64)
+	testing.expect_value(t, count_byte(data, 0), 64)
+	mem.free_bytes(data, allocator)
+}
+
+@(test)
+test_tlsf_resize :: proc(t: ^testing.T) {
+	alloc: tlsf.Allocator
+	defer tlsf.destroy(&alloc)
+	if err := tlsf.init_from_allocator(&alloc, context.allocator, 1 << 16); err != .None {
+		testing.fail_now(t, "TLSF init error")
+	}
+
+	expect_resize_keeps_bytes(t, tlsf.allocator(&alloc))
+	expect_resize_realigns(t, tlsf.allocator(&alloc))
+}
+
+@(require_results)
+init_virtual_arena :: proc(arena: ^virtual.Arena, kind: virtual.Arena_Kind, buffer: []byte) -> mem.Allocator_Error {
+	switch kind {
+	case .Growing: return virtual.arena_init_growing(arena)
+	case .Static:  return virtual.arena_init_static(arena)
+	case .Buffer:  return virtual.arena_init_buffer(arena, buffer)
+	}
+	return nil
+}
+
+@(test)
+test_virtual_arena_resize :: proc(t: ^testing.T) {
+	for kind in virtual.Arena_Kind {
+		buf: [8192]byte
+		arena: virtual.Arena
+		testing.expect_value(t, init_virtual_arena(&arena, kind, buf[:]), nil)
+		defer virtual.arena_destroy(&arena)
+
+		expect_resize_keeps_bytes(t, virtual.arena_allocator(&arena))
+		expect_resize_realigns(t, virtual.arena_allocator(&arena))
+	}
+}
+
+@(test)
+test_scratch_resize_to_end :: proc(t: ^testing.T) {
+	s: mem.Scratch
+	mem.scratch_init(&s, 1024)
+	defer mem.scratch_destroy(&s)
+	allocator := mem.scratch_allocator(&s)
+
+	data, _ := mem.alloc_bytes(64, 16, allocator)
+	room := len(s.data) - int(uintptr(raw_data(data)) - uintptr(raw_data(s.data)))
+
+	grown, err := mem.resize_bytes(data, room, 16, allocator)
+	testing.expect_value(t, err, nil)
+	testing.expect(t, raw_data(grown) == raw_data(data), "the last allocation did not grow in place")
+}
+
+@(test)
+test_arena_resize_in_place :: proc(t: ^testing.T) {
+	buf: [1024]byte
+	mem.set(&buf, 0xCC, len(buf))
+	arena: mem.Arena
+	mem.arena_init(&arena, buf[:])
+	allocator := mem.arena_allocator(&arena)
+
+	first, _ := mem.alloc_bytes(64, 16, allocator)
+	last,  _ := mem.alloc_bytes(64, 16, allocator)
+	mem.set(raw_data(last), 0xAA, len(last))
+	offset := arena.offset
+
+	grown, err := mem.resize_bytes(last, 128, 16, allocator)
+	testing.expect_value(t, err, nil)
+	testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place")
+	testing.expect_value(t, arena.offset,    offset+64)
+	testing.expect_value(t, arena.peak_used, offset+64)
+	testing.expect_value(t, count_byte(grown[:64], 0xAA), 64)
+	testing.expect_value(t, count_byte(grown[64:], 0),    64)
+
+	shrunk, shrunk_err := mem.resize_bytes(grown, 32, 16, allocator)
+	testing.expect_value(t, shrunk_err, nil)
+	testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place")
+	testing.expect_value(t, arena.offset, offset-32)
+
+	// An earlier allocation shrinks in place but keeps its tail, and moves to grow.
+	offset = arena.offset
+	first_shrunk, _ := mem.resize_bytes(first, 16, 16, allocator)
+	testing.expect(t, raw_data(first_shrunk) == raw_data(first), "an earlier allocation did not shrink in place")
+	testing.expect_value(t, arena.offset, offset)
+	first_grown, _ := mem.resize_bytes(first_shrunk, 128, 16, allocator)
+	testing.expect(t, raw_data(first_grown) != raw_data(first), "an earlier allocation grew in place over a later one")
+	testing.expect_value(t, count_byte(shrunk, 0xAA), 32)
+
+	// While a temporary memory region is active, a shrunk last allocation keeps its tail.
+	offset = arena.offset
+	tmp := mem.begin_arena_temp_memory(&arena)
+	_, _ = mem.resize_bytes(first_grown, 64, 16, allocator)
+	testing.expect_value(t, arena.offset, offset)
+	mem.end_arena_temp_memory(tmp)
+	testing.expect_value(t, arena.offset, offset)
+}
+
+@(test)
+test_small_stack_resize_in_place :: proc(t: ^testing.T) {
+	buf: [1024]byte
+	s: mem.Small_Stack
+	mem.small_stack_init(&s, buf[:])
+	allocator := mem.small_stack_allocator(&s)
+
+	first, _ := mem.alloc_bytes(64, 16, allocator)
+	last,  _ := mem.alloc_bytes(64, 16, allocator)
+	mem.set(raw_data(last), 0xAA, len(last))
+	offset := s.offset
+
+	grown, err := mem.resize_bytes(last, 128, 16, allocator)
+	testing.expect_value(t, err, nil)
+	testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place")
+	testing.expect_value(t, s.offset,    offset+64)
+	testing.expect_value(t, s.peak_used, offset+64)
+
+	shrunk, shrunk_err := mem.resize_bytes(grown, 32, 16, allocator)
+	testing.expect_value(t, shrunk_err, nil)
+	testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place")
+	testing.expect_value(t, s.offset, offset-32)
+
+	// An earlier allocation shrinks in place, and moves to grow.
+	offset = s.offset
+	first_shrunk, _ := mem.resize_bytes(first, 16, 16, allocator)
+	testing.expect(t, raw_data(first_shrunk) == raw_data(first), "an earlier allocation did not shrink in place")
+	testing.expect_value(t, s.offset, offset)
+	first_grown, _ := mem.resize_bytes(first_shrunk, 128, 16, allocator)
+	testing.expect(t, raw_data(first_grown) != raw_data(first), "an earlier allocation grew in place over a later one")
+	testing.expect_value(t, count_byte(shrunk, 0xAA), 32)
+}
+
+@(test)
+test_dynamic_arena_resize_in_place :: proc(t: ^testing.T) {
+	arena: mem.Dynamic_Arena
+	mem.dynamic_arena_init(&arena)
+	defer mem.dynamic_arena_destroy(&arena)
+	allocator := mem.dynamic_arena_allocator(&arena)
+
+	last, _ := mem.alloc_bytes(64, 16, allocator)
+	mem.set(raw_data(last), 0xAA, len(last))
+	bytes_left := arena.bytes_left
+
+	grown, err := mem.resize_bytes(last, 128, 16, allocator)
+	testing.expect_value(t, err, nil)
+	testing.expect(t, raw_data(grown) == raw_data(last), "the last allocation did not grow in place")
+	testing.expect_value(t, arena.bytes_left, bytes_left-64)
+	testing.expect_value(t, count_byte(grown[:64], 0xAA), 64)
+	testing.expect_value(t, count_byte(grown[64:], 0),    64)
+
+	shrunk, shrunk_err := mem.resize_bytes(grown, 32, 16, allocator)
+	testing.expect_value(t, shrunk_err, nil)
+	testing.expect(t, raw_data(shrunk) == raw_data(last), "the last allocation did not shrink in place")
+	testing.expect_value(t, arena.bytes_left, bytes_left+32)
+
+	// An allocation followed by another is not the last one, even when a
+	// stricter alignment would round its size up to reach past the later one.
+	first: []byte
+	for {
+		first, _ = mem.alloc_bytes(16, 16, allocator)
+		if mem.is_aligned(raw_data(first), 64) {
+			break
+		}
+	}
+	later, _ := mem.alloc_bytes(48, 16, allocator)
+	mem.set(raw_data(later), 0xBB, len(later))
+
+	first_grown, _ := mem.resize_bytes(first, 64, 64, allocator)
+	testing.expect(t, raw_data(first_grown) != raw_data(first), "an earlier allocation grew in place over a later one")
+	testing.expect_value(t, count_byte(later, 0xBB), 48)
+}
+
+@(test)
+test_virtual_arena_resize_in_place :: proc(t: ^testing.T) {
+	for kind in virtual.Arena_Kind {
+		buf: [16384]byte
+		arena: virtual.Arena
+		testing.expect_value(t, init_virtual_arena(&arena, kind, buf[:]), nil)
+		defer virtual.arena_destroy(&arena)
+		allocator := virtual.arena_allocator(&arena)
+
+		data, _ := mem.alloc_bytes(4096, 16, allocator)
+		mem.set(raw_data(data), 0xAA, len(data))
+		used  := arena.curr_block.used
+		total := arena.total_used
+
+		// Shrinking the last allocation gives its tail back, zeroed.
+		shrunk, err := mem.resize_bytes(data, 1000, 16, allocator)
+		testing.expect_value(t, err, nil)
+		testing.expect(t, raw_data(shrunk) == raw_data(data), "the last allocation did not shrink in place")
+		testing.expect_value(t, count_byte(shrunk, 0xAA), 1000)
+		testing.expect_value(t, arena.curr_block.used, used-3096)
+		testing.expect_value(t, arena.total_used,      total-3096)
+
+		next, _ := mem.alloc_bytes(3000, 1, allocator)
+		testing.expect(t, raw_data(next) == raw_data(shrunk[1000:]), "the tail was not given back")
+		testing.expect_value(t, count_byte(next, 0), 3000)
+
+		// An earlier allocation shrinks in place, keeping its tail.
+		used = arena.curr_block.used
+		_, _ = mem.resize_bytes(shrunk, 500, 16, allocator)
+		testing.expect_value(t, arena.curr_block.used, used)
+	}
+}
+
+@(test)
+test_virtual_arena_shrink_in_temp :: proc(t: ^testing.T) {
+	for kind in virtual.Arena_Kind {
+		buf: [16384]byte
+		arena: virtual.Arena
+		testing.expect_value(t, init_virtual_arena(&arena, kind, buf[:]), nil)
+		defer virtual.arena_destroy(&arena)
+		allocator := virtual.arena_allocator(&arena)
+
+		data, _ := mem.alloc_bytes(4096, 16, allocator)
+		used := arena.curr_block.used
+
+		// The temporary region saved the end of the allocation, so it keeps its tail.
+		temp := virtual.arena_temp_begin(&arena)
+		shrunk, err := mem.resize_bytes(data, 1000, 16, allocator)
+		testing.expect_value(t, err, nil)
+		testing.expect(t, raw_data(shrunk) == raw_data(data), "the last allocation did not shrink in place")
+		testing.expect_value(t, arena.curr_block.used, used)
+		virtual.arena_temp_end(temp)
+		testing.expect_value(t, arena.curr_block.used, used)
+	}
+}