diff --git a/core/diff/compact.odin b/core/diff/compact.odin new file mode 100644 index 000000000..b502bc6e7 --- /dev/null +++ b/core/diff/compact.odin @@ -0,0 +1,443 @@ +#+private +package diff + +/* + The `.Compact` post-pass. It slides each group of changes along equal tokens to where git puts it: beside a + change on the other side if it can, otherwise as far down as it goes. With `.Indent_Heuristic` it picks a + position by the indentation and blank lines around it instead. With `.Histogram` each group that moved or grew + is diffed again against its partner. + + A group is a maximal run of changed tokens on one side, and may be empty. A side with `u` unchanged tokens has + `u + 1` groups: one before each unchanged token, and one after the last. The `k`-th group of one side is the + partner of the `k`-th group of the other. Both sit between the same pair of unchanged tokens. +*/ + +// The indent of a line of only whitespace. +LINE_INDENT_BLANK :: 255 + +// An indent is never measured past this. +LINE_INDENT_MAX :: 200 + +// A tab moves to the next multiple of this. +LINE_INDENT_TAB :: 8 + +// The indent heuristic tries a group at most this many positions above the bottom. +COMPACT_MAX_SLIDING :: 100 + +// The indent heuristic looks past at most this many blank lines. +COMPACT_MAX_BLANKS :: 20 + +// The weights of the indent heuristic. A lower score is a better place to split. +INDENT_START_OF_FILE :: 1 +INDENT_END_OF_FILE :: 21 +INDENT_BLANKS :: -30 // for each blank line around the split +INDENT_BLANKS_AFTER :: 6 // for each blank line after the split +INDENT_DEEPER :: -4 // indented more than above +INDENT_DEEPER_BLANKS :: 10 +INDENT_BLOCK_START :: 24 // indented less than above, but more below +INDENT_BLOCK_START_BLANKS :: 17 +INDENT_BLOCK_END :: 23 // indented less than above +INDENT_BLOCK_END_BLANKS :: 17 +INDENT_EFFECTIVE_WEIGHT :: 60 // for the sign of a difference in effective indent + +// The options `rediff` diffs with, apart from `max_cost`. +REDIFF_OPTIONS :: Options{ + algorithm = .Myers, + flags = {.Prune}, +} + +// One side of a diff, for `.Compact`. +Compact_Side :: struct { + ids: []i32, + offset: int, // where this side's bits start in `Changes.bits` + indents: []u8, // one per token, for `.Indent_Heuristic`; nil otherwise +} + +// The changed tokens `[start, end)` of one side. +Group :: struct { + start, end: int, +} + +// A group of one side and its partner, walked through both sides in step. +Compact_Walk :: struct { + c: Changes, + this, other: Compact_Side, + g, go: Group, +} + +// How good a place to split the indent heuristic finds. Lower is better. +Indent_Score :: struct { + effective_indent: int, + penalty: int, +} + + +/* + Measures the indent of `line`, its final "\n" included. A space adds 1, a tab moves to the next multiple of + `LINE_INDENT_TAB`, and the other whitespace adds nothing. Returns at most `LINE_INDENT_MAX`, or + `LINE_INDENT_BLANK` for a line of only whitespace. +*/ +@(require_results) +line_indent :: proc "contextless" (line: string) -> u8 { + indent := 0 + for i in 0..= LINE_INDENT_MAX { + return LINE_INDENT_MAX + } + } + return LINE_INDENT_BLANK +} + +// Compacts `a`'s side against `b`, then `b`'s side against `a`. +compact :: proc(a, b: []i32, indents: []u8, id_count: int, changes: Changes, s: ^Scratch, opts: Options, loc := #caller_location) { + a_side := Compact_Side{a, 0, nil} + b_side := Compact_Side{b, _inserted_offset(changes), nil} + if indents != nil { + assert(len(indents) == len(a) + len(b), "an indent is needed for every line", loc) + a_side.indents = indents[:len(a)] + b_side.indents = indents[len(a):] + } + + // Diffing merged groups again is git's for histogram only + rediffing := opts.algorithm == .Histogram + compact_side(changes, a_side, b_side, rediffing, id_count, opts.max_cost, s, loc) + compact_side(changes, b_side, a_side, rediffing, id_count, opts.max_cost, s, loc) +} + +// The scratch `rediff` takes for any groups within `n` by `m` tokens. +@(require_results) +rediff_scratch_size :: proc "contextless" (n, m: int, id_count: int) -> int { + return scratch_bytes(u64, changes_size(n, m)) + prune_scratch_size(n, m, id_count, REDIFF_OPTIONS) +} + +/* + Diffs `this.ids[g.start:g.end]` against `other.ids[go.start:go.end]` again, with Myers after `.Prune`'s + preparation. Every bit of both groups is then set or cleared: `this` takes the deletions, and `other` the + insertions. Gives back its scratch. +*/ +rediff :: proc(c: Changes, this, other: Compact_Side, g, go: Group, id_count: int, max_cost: int, s: ^Scratch, loc := #caller_location) { + // Copies `count` bits from `src[src_lo:]` to `dst[dst_lo:]` + copy_bits :: proc "contextless" (dst: []u64, dst_lo: int, src: []u64, src_lo: int, count: int) { + for t in 0..= 0: + // Keep a replacement together, rather than a deletion apart from an insertion + for w.go.start == w.go.end { + compact_slide_up(&w) + } + + case this.indents != nil: + target := indent_best_end(this.indents, w.g, earliest_end) + for w.g.end > target { + compact_slide_up(&w) + } + } + + if rediffing && w.go.start < w.go.end && w.g != was { + rediff(c, this, other, w.g, w.go, id_count, max_cost, s, loc) + } + } + + compact_next_group(c, this, &w.g) or_break + partner_moved := compact_next_group(c, other, &w.go) + assert(partner_moved, "changes have more unchanged tokens on one side", loc) + } + assert(w.go.end == len(other.ids), "changes have more unchanged tokens on one side", loc) +} + +compact_put_bit :: #force_inline proc "contextless" (bits: []u64, i: int, set: bool) { + mask := u64(1) << uint(i & 63) + if set { + bits[i >> 6] |= mask + } else { + bits[i >> 6] &~= mask + } +} + +// The group before the first unchanged token of `side`. +@(require_results) +compact_first_group :: proc "contextless" (c: Changes, side: Compact_Side) -> Group { + return {0, _bit_run(c.bits, side.offset, side.offset + len(side.ids), true)} +} + +// Moves `g` to the next group of `side`, just past the unchanged token that ends it. False at the last group. +@(require_results) +compact_next_group :: proc "contextless" (c: Changes, side: Compact_Side, g: ^Group) -> bool { + n := len(side.ids) + if g.end >= n { + return false + } + g.start = g.end + 1 + g.end = g.start + _bit_run(c.bits, side.offset + g.start, side.offset + n, true) + return true +} + +// Moves `g` to the previous group of `side`, just before the unchanged token that starts it. False at the first. +@(require_results) +compact_prev_group :: proc "contextless" (c: Changes, side: Compact_Side, g: ^Group) -> bool { + if g.start <= 0 { + return false + } + g.end = g.start - 1 + g.start = compact_run_start(c, side, g.end) + return true +} + +// Where the run of changed tokens ending just before `end` starts. +@(require_results) +compact_run_start :: proc "contextless" (c: Changes, side: Compact_Side, end: int) -> (start: int) { + start = end + for start > 0 && bit_get(c.bits, side.offset + start - 1) { + start -= 1 + } + return +} + +@(require_results) +compact_can_slide_down :: proc "contextless" (w: ^Compact_Walk) -> bool { + ids := w.this.ids + return w.g.start < w.g.end && + w.g.end < len(ids) && + ids[w.g.end] == ids[w.g.start] +} + +@(require_results) +compact_can_slide_up :: proc "contextless" (w: ^Compact_Walk) -> bool { + ids := w.this.ids + return w.g.start < w.g.end && + w.g.start > 0 && + ids[w.g.start-1] == ids[w.g.end-1] +} + +// Slides the group down one token, taking in any group it then touches. The partner moves to the next group. +compact_slide_down :: proc "contextless" (w: ^Compact_Walk) { + offset := w.this.offset + n := len(w.this.ids) + compact_put_bit(w.c.bits, offset + w.g.start, false) + compact_put_bit(w.c.bits, offset + w.g.end, true) + w.g.start += 1 + w.g.end += 1 + w.g.end += _bit_run(w.c.bits, offset + w.g.end, offset + n, true) + + moved := compact_next_group(w.c, w.other, &w.go) + assert_contextless(moved, "changes have more unchanged tokens on one side") +} + +// Slides the group up one token, taking in any group it then touches. The partner moves to the previous group. +compact_slide_up :: proc "contextless" (w: ^Compact_Walk) { + offset := w.this.offset + compact_put_bit(w.c.bits, offset + w.g.start - 1, true) + compact_put_bit(w.c.bits, offset + w.g.end - 1, false) + w.g.end -= 1 + w.g.start = compact_run_start(w.c, w.this, w.g.start) + + moved := compact_prev_group(w.c, w.other, &w.go) + assert_contextless(moved, "changes have more unchanged tokens on one side") +} + + +/* + The indent heuristic. Tries the group `g`, at the bottom, at each end from up to `COMPACT_MAX_SLIDING` above + down to where it is, and returns the end that scores best. Ties go to the later, lower end. +*/ +@(require_results) +indent_best_end :: proc "contextless" (indents: []u8, g: Group, earliest_end: int) -> (best_end: int) { + size := g.end - g.start + best := Indent_Score{} + best_end = -1 + for end := max(earliest_end, g.end - size - 1, g.end - COMPACT_MAX_SLIDING); end <= g.end; end += 1 { + score := Indent_Score{} + indent_score_split(indents, end, &score) + indent_score_split(indents, end - size, &score) + if best_end < 0 || indent_compare(score, best) <= 0 { + best = score + best_end = end + } + } + return +} + +// Negative when `x` is better than `y`, and zero when they are as good. +@(require_results) +indent_compare :: proc "contextless" (x, y: Indent_Score) -> int { + sign := 0 + switch { + case x.effective_indent > y.effective_indent: sign = 1 + case x.effective_indent < y.effective_indent: sign = -1 + } + return INDENT_EFFECTIVE_WEIGHT*sign + (x.penalty - y.penalty) +} + +/* + Adds to `score` for a split just above line `s`, from the indents of the lines around it. A blank line has + indent -1, and so does the line past the end. +*/ +indent_score_split :: proc "contextless" (indents: []u8, s: int, score: ^Indent_Score) { + @(require_results) + indent_of :: #force_inline proc "contextless" (x: u8) -> int { + if x == LINE_INDENT_BLANK { + return -1 + } + return int(x) + } + + + n := len(indents) + + // The line itself + end_of_file := s >= n + indent := -1 + if !end_of_file { + indent = indent_of(indents[s]) + } + + // The first line above that is not blank, looking past `COMPACT_MAX_BLANKS` blank lines at most + pre_blank := 0 + pre_indent := -1 + for i := s - 1; i >= 0; i -= 1 { + if indents[i] != LINE_INDENT_BLANK { + pre_indent = int(indents[i]) + break + } + pre_blank += 1 + if pre_blank == COMPACT_MAX_BLANKS { + pre_indent = 0 + break + } + } + + // The same below, from the line after it + post_blank := 0 + post_indent := -1 + for i := s + 1; i < n; i += 1 { + if indents[i] != LINE_INDENT_BLANK { + post_indent = int(indents[i]) + break + } + post_blank += 1 + if post_blank == COMPACT_MAX_BLANKS { + post_indent = 0 + break + } + } + + if pre_indent == -1 && pre_blank == 0 { + score.penalty += INDENT_START_OF_FILE + } + if end_of_file { + score.penalty += INDENT_END_OF_FILE + } + + // The blank lines after the split, counting the line itself + post := 0 + if indent == -1 { + post = 1 + post_blank + } + total := pre_blank + post + score.penalty += INDENT_BLANKS*total + INDENT_BLANKS_AFTER*post + + ind := indent + if indent == -1 { + ind = post_indent + } + score.effective_indent += ind + + blanks := total != 0 + switch { + case ind == -1 || pre_indent == -1 || ind == pre_indent: + // Nothing to say + + case ind > pre_indent: + score.penalty += blanks ? INDENT_DEEPER_BLANKS : INDENT_DEEPER + + case post_indent != -1 && post_indent > ind: + score.penalty += blanks ? INDENT_BLOCK_START_BLANKS : INDENT_BLOCK_START + + case: + score.penalty += blanks ? INDENT_BLOCK_END_BLANKS : INDENT_BLOCK_END + } +} diff --git a/core/diff/diff.odin b/core/diff/diff.odin new file mode 100644 index 000000000..d855bd88b --- /dev/null +++ b/core/diff/diff.odin @@ -0,0 +1,422 @@ +package diff + +import "base:intrinsics" +import "base:runtime" + +// The algorithm used to compute a diff. +Algorithm :: enum u8 { + Myers, // Myers O(ND) with a cost cap; close to minimal (default) + Minimal, // Myers with no cost cap; always minimal, but can be slow on very different inputs + Histogram, // anchors on rare tokens; falls back to Myers + Ratcliff_Obershelp, // longest common block first; not minimal; falls back to Myers at the cost cap +} + +Option_Flag :: enum u8 { + Prune, // pre-pass: set aside the common ends, tokens with no match, and runs of very frequent tokens + Compact, // post-pass: slide each group of changes to where git puts it + Indent_Heuristic, // with .Compact, for line diffs of text: slide each group by the indentation around it +} +Option_Flags :: bit_set[Option_Flag; u8] + +Options :: struct { + algorithm: Algorithm, + flags: Option_Flags, + max_cost: int, // 0 means the default heuristic; ignored by `.Minimal` +} + +/* + The result of a diff: one bit per token on each side. + + A set bit `i` on the `a` side means `a[i]` was deleted. + A set bit `j` on the `b` side means `b[j]` was inserted. + The `a` side is the first `(n+63)/64` words of `bits`; the `b` side is the rest. +*/ +Changes :: struct { + bits: []u64, + n, m: int, +} + +// Counts of tokens by what happened to them. +Stats :: struct { + equal, deleted, inserted: int, +} + + +// Returns the number of words `Changes` needs for inputs of length `n` and `m`. +@(require_results) +changes_size :: proc "contextless" (n, m: int) -> (words: int) { + return bit_words(n) + bit_words(m) +} + +// Makes `Changes` from a caller-provided buffer of at least `changes_size(n, m)` words. +@(require_results) +changes_from_buffer :: proc(n, m: int, bits: []u64, loc := #caller_location) -> Changes { + words := changes_size(n, m) + assert(len(bits) >= words, "buffer too small for the changes", loc) + return {bits[:words], n, m} +} + +@(require_results) +changes_make :: proc(n, m: int, allocator: runtime.Allocator, loc := #caller_location) -> (c: Changes, err: runtime.Allocator_Error) { + bits := make([]u64, changes_size(n, m), allocator, loc) or_return + return {bits, n, m}, nil +} + +// Frees `c` with the allocator it was made with. +changes_destroy :: proc(c: Changes, allocator: runtime.Allocator, loc := #caller_location) { + delete(c.bits, allocator, loc) +} + +@(require_results) +is_deleted :: #force_inline proc "contextless" (c: Changes, i: int) -> bool { + return bit_get(c.bits, i) +} + +@(require_results) +is_inserted :: #force_inline proc "contextless" (c: Changes, j: int) -> bool { + return bit_get(c.bits, _inserted_offset(c) + j) +} + +@(require_results) +stats :: proc "contextless" (c: Changes) -> (s: Stats) { + a_words := bit_words(c.n) + for w in c.bits[:a_words] { + s.deleted += int(intrinsics.count_ones(w)) + } + for w in c.bits[a_words:] { + s.inserted += int(intrinsics.count_ones(w)) + } + s.equal = c.n - s.deleted + return +} + +// Returns `2*equal / (n+m)`, from 0 (nothing in common) to 1 (identical); 1 when both inputs are empty. +@(require_results) +similarity :: proc "contextless" (c: Changes) -> f64 { + if c.n + c.m == 0 { + return 1 + } + s := stats(c) + return 2*f64(s.equal) / f64(c.n + c.m) +} + + +/* + Core procedures: these never allocate. + + `scratch` may have any alignment. The scratch sizes include slack to align it. + A buffer smaller than the size asked for is an assertion failure, not an error. +*/ + +/* + Returns the scratch bytes `diff_buffer` and `diff_by_buffer` need for inputs of length `n` and `m`. + + The token IDs last the whole diff. The interning table and the algorithm share the rest, one after the other. +*/ +@(require_results) +scratch_size :: proc "contextless" (n, m: int, opts := Options{}) -> int { + tokens := n + m + return SCRATCH_ALIGN + scratch_bytes(i32, tokens) + max(intern_scratch_size(tokens), algorithm_scratch_size(n, m, tokens, opts)) +} + +// Returns the scratch bytes `diff_interned_buffer` needs for already-interned inputs. +@(require_results) +scratch_size_interned :: proc "contextless" (n, m: int, id_count: int, opts := Options{}) -> int { + return SCRATCH_ALIGN + algorithm_scratch_size(n, m, id_count, opts) +} + +// Diffs already-interned inputs: equal tokens share an ID, and every ID is in `[0, id_count)`. +diff_interned_buffer :: proc(a, b: []i32, id_count: int, changes: Changes, scratch: []byte, opts := Options{}, loc := #caller_location) { + // Positions are kept as `i32`. Interning keeps them the same way + assert(len(a) + len(b) < int(max(i32)), "too many tokens to diff", loc) + + s := Scratch{buf = scratch} + diff_interned_scratch(a, b, nil, id_count, changes, &s, opts, loc) +} + +diff_buffer :: proc(a, b: []$T, changes: Changes, scratch: []byte, opts := Options{}, loc := #caller_location) where intrinsics.type_is_comparable(T) { + equal :: proc(x, y: T, _: rawptr) -> bool { + return x == y + } + hash :: proc(x: T, _: rawptr) -> u64 { + x := x + return u64(intrinsics.type_hasher_proc(T)(&x, 0)) + } + diff_by_buffer(a, b, equal, hash, nil, changes, scratch, opts, loc) +} + +// `equal` and `hash` must agree: equal tokens must hash the same. +diff_by_buffer :: proc( + a, b: []$T, + equal: proc(x, y: T, user_data: rawptr) -> bool, + hash: proc(x: T, user_data: rawptr) -> u64, + user_data: rawptr, + changes: Changes, + scratch: []byte, + opts := Options{}, + loc := #caller_location, +) { + n, m := len(a), len(b) + assert(changes.n == n && changes.m == m, "changes sized for different inputs", loc) + assert(n + m < int(max(i32)), "too many tokens to intern", loc) + + s := Scratch{buf = scratch} + ids := scratch_take(&s, i32, n + m, loc) + + table_start := s.pos + table := scratch_take(&s, Intern_Slot, intern_table_len(n + m), loc) + id_count := intern(a, b, T, equal, hash, user_data, ids, table) + s.pos = table_start + + diff_interned_scratch(ids[:n], ids[n:], nil, id_count, changes, &s, opts, loc) +} + + +/* + Allocating procedures. + + Each takes its allocators explicitly: + - `allocator` gives the returned memory. Free it with the same allocator. + - `transient_allocator` gives the scratch. It lives only for the call. + + The result is allocated first and the scratch last. The scratch is freed before returning, even when + allocating it fails. Heap allocators never leak. Allocators that free in reverse order (e.g. `mem.Stack`) + work when passed as both. + + One bump arena passed as both cannot free the scratch. It stays until the arena is reset. Use separate + arenas for the two lifetimes, or a `*_buffer` procedure with scratch from a temporary region of the arena. +*/ + +@(require_results) +diff :: proc( + a, b: []$T, + allocator: runtime.Allocator, + transient_allocator: runtime.Allocator, + opts := Options{}, + loc := #caller_location, +) -> (c: Changes, err: runtime.Allocator_Error) where intrinsics.type_is_comparable(T) { + scratch: []byte + c, scratch = _changes_and_scratch_make( + len(a), + len(b), + scratch_size(len(a), len(b), opts), + allocator, + transient_allocator, + loc, + ) or_return + diff_buffer(a, b, c, scratch, opts, loc) + delete(scratch, transient_allocator, loc) + return +} + +@(require_results) +diff_by :: proc( + a, b: []$T, + equal: proc(x, y: T, user_data: rawptr) -> bool, + hash: proc(x: T, user_data: rawptr) -> u64, + user_data: rawptr, + allocator: runtime.Allocator, + transient_allocator: runtime.Allocator, + opts := Options{}, + loc := #caller_location, +) -> (c: Changes, err: runtime.Allocator_Error) { + scratch: []byte + c, scratch = _changes_and_scratch_make( + len(a), + len(b), + scratch_size(len(a), len(b), opts), + allocator, + transient_allocator, + loc, + ) or_return + diff_by_buffer(a, b, equal, hash, user_data, c, scratch, opts, loc) + delete(scratch, transient_allocator, loc) + return +} + +@(require_results) +diff_interned :: proc( + a, b: []i32, + id_count: int, + allocator: runtime.Allocator, + transient_allocator: runtime.Allocator, + opts := Options{}, + loc := #caller_location, +) -> (c: Changes, err: runtime.Allocator_Error) { + scratch: []byte + c, scratch = _changes_and_scratch_make( + len(a), len(b), + scratch_size_interned(len(a), len(b), id_count, opts), + allocator, + transient_allocator, + loc, + ) or_return + diff_interned_buffer(a, b, id_count, c, scratch, opts, loc) + delete(scratch, transient_allocator, loc) + return +} + + +/* + Scratch for diffing interned tokens: the `.Prune` pre-pass, if asked for, and the algorithm. + `.Compact` runs once they are done. Its scratch for `.Histogram` reuses theirs. +*/ +@(private, require_results) +algorithm_scratch_size :: proc "contextless" (n, m: int, id_count: int, opts: Options) -> int { + size := unpruned_scratch_size(n, m, id_count, opts) + if .Prune in opts.flags { + size = prune_scratch_size(n, m, id_count, opts) + } + if .Compact in opts.flags && opts.algorithm == .Histogram { + size = max(size, rediff_scratch_size(n, m, id_count)) + } + return size +} + +// Scratch for the algorithm alone. +@(private, require_results) +unpruned_scratch_size :: proc "contextless" (n, m: int, id_count: int, opts: Options) -> int { + switch opts.algorithm { + case .Myers, .Minimal: + return myers_scratch_size(n, m) + case .Histogram: + return histogram_scratch_size(n, m, id_count) + case .Ratcliff_Obershelp: + return ratcliff_obershelp_scratch_size(n, m, id_count) + } + return 0 +} + +// `indents` holds the indent of each line of `a` then `b`, for `.Indent_Heuristic`; only text has them. +@(private) +diff_interned_scratch :: proc( + a, b: []i32, + indents: []u8, + id_count: int, + changes: Changes, + s: ^Scratch, + opts: Options, + loc := #caller_location, +) { + assert(changes.n == len(a), "changes sized for different inputs", loc) + assert(changes.m == len(b), "changes sized for different inputs", loc) + assert(len(changes.bits) >= changes_size(changes.n, changes.m), "buffer too small for the changes", loc) + assert(id_count >= 0, "negative id_count", loc) + + if .Indent_Heuristic in opts.flags { + assert(.Compact in opts.flags, ".Indent_Heuristic only applies with .Compact", loc) + assert(indents != nil, ".Indent_Heuristic needs a line diff of text", loc) + } + + for &w in changes.bits { + w = 0 + } + + start := s.pos + if .Prune in opts.flags { + prune(a, b, id_count, changes, s, opts, loc) + } else { + run_algorithm(a, b, id_count, changes, s, opts, loc) + } + + // The scratch of what ran before is free again + if .Compact in opts.flags { + s.pos = start + compact(a, b, indents, id_count, changes, s, opts, loc) + } +} + +// Runs the chosen algorithm into `changes`. `changes` must already be zeroed. +@(private) +run_algorithm :: proc(a, b: []i32, id_count: int, changes: Changes, s: ^Scratch, opts: Options, loc := #caller_location) { + switch opts.algorithm { + case .Myers, .Minimal: + myers(a, b, changes, s, opts.algorithm == .Minimal, opts.max_cost, loc) + case .Histogram: + histogram(a, b, id_count, changes, s, opts.max_cost, loc) + case .Ratcliff_Obershelp: + ratcliff_obershelp(a, b, id_count, changes, s, opts.max_cost, loc) + } +} + +// Allocates the result first, then the scratch; if the scratch fails, the result is freed again. +@(require_results) +_changes_and_scratch_make :: proc( + n, m: int, + scratch_len: int, + allocator: runtime.Allocator, + transient_allocator: runtime.Allocator, + loc := #caller_location, +) -> (c: Changes, scratch: []byte, err: runtime.Allocator_Error) { + + c = changes_make(n, m, allocator, loc) or_return + scratch, err = make([]byte, scratch_len, transient_allocator, loc) + if err != nil { + changes_destroy(c, allocator, loc) + c = {} + } + return +} + + +@(private, require_results) +bit_words :: #force_inline proc "contextless" (n: int) -> int { + return (n + 63) >> 6 +} + +// Where the bits of `b` start in `c.bits`: after those of `a`, rounded up to a whole word. +@(require_results) +_inserted_offset :: #force_inline proc "contextless" (c: Changes) -> int { + return bit_words(c.n)*64 +} + +@(private, require_results) +bit_get :: #force_inline proc "contextless" (bits: []u64, i: int) -> bool { + return bits[i >> 6] & (u64(1) << uint(i & 63)) != 0 +} + +// Marks `a[lo:hi]` as deleted. +_mark_deleted :: proc "contextless" (c: Changes, lo, hi: int) { + _set_bit_range(c.bits, lo, hi) +} + +// Marks `b[lo:hi]` as inserted. +_mark_inserted :: proc "contextless" (c: Changes, lo, hi: int) { + offset := _inserted_offset(c) + _set_bit_range(c.bits, offset + lo, offset + hi) +} + +_set_bit_range :: proc "contextless" (bits: []u64, lo, hi: int) { + for i in lo..> 6] |= mask + } +} + +// Returns how many bits from `lo` up to `hi` are all set, or all clear when `set` is false. +@(require_results) +_bit_run :: proc "contextless" (bits: []u64, lo, hi: int, set: bool) -> int { + for i := lo; i < hi; /**/ { + w := bits[i >> 6] + if !set { + w = ~w + } + + shift := uint(i & 63) + left := 64 - int(shift) + + // A 1 for each bit that differs including the bits shifted in at the top + miss := ~(w >> shift) + + run := 64 + if miss != 0 { + run = int(intrinsics.count_trailing_zeros(miss)) + } + + if run < left { + return min(i + run, hi) - lo + } + i += left + } + return hi - lo +} \ No newline at end of file diff --git a/core/diff/doc.odin b/core/diff/doc.odin new file mode 100644 index 000000000..baed3bbd3 --- /dev/null +++ b/core/diff/doc.odin @@ -0,0 +1,216 @@ +/* +Differences between two sequences: which tokens of `a` were deleted and which tokens of `b` were inserted. + +Diffs slices of any comparable type, slices of any type with your own equality and hash, and text by lines, +words, or runes. Read the result as edits, hunks, or substrings. Write it as a unified diff, in colour if +wanted, with the changed parts of lines highlighted. + + +## The Result + +A diff is a `Changes`: one bit per token of `a` (deleted or not) and per token of `b` (inserted or not). +Its size is `changes_size(n, m)` words, known before diffing. Everything else reads from it without allocating: + + edit_iterate `Edit`s: runs of equal, deleted, and inserted tokens + hunk_iterate `Hunk`s: groups of changes with equal tokens around them + span_iterate the same runs as substrings of a `Text_Diff` + unified_iterate the lines of a unified diff of a line diff + write_unified a unified diff, written to an `io.Writer` + stats the number of equal, deleted, and inserted tokens + + +## Algorithms + + .Myers Myers' O(ND) diff with git's cost cap. Close to minimal. The default. + .Minimal Myers without the cap. Always minimal. Slow on very different inputs. + .Histogram git's histogram diff. Anchors on the rarest tokens, keeping moved blocks whole. + Not minimal. + .Ratcliff_Obershelp The longest common block first, then each side of it. Not minimal. Tokens occurring + in `b` more than 64 times (e.g. `}` or blank lines) extend blocks but never start + them. Hands over to Myers past a budget of work. + +`Options.max_cost` caps Myers in edits, wherever Myers runs. For `.Ratcliff_Obershelp`, it is also the +budget in steps per token. 0 means the default. + +Flags: + + .Prune Before diffing, set aside the common prefix and suffix, tokens absent from the other + side, and runs of very frequent tokens. Often much faster. + .Compact After diffing, slide each group of changes to where git puts it: beside a change on + the other side, otherwise as far down as it goes. With `.Histogram`, merged groups + are diffed again. + .Indent_Heuristic With `.Compact`, slide each group to where the indentation and blank lines around it + read best. Line diffs of text only. Anything else is an assertion failure. + +These match git exactly: + + .Myers + .Prune, .Compact, and .Indent_Heuristic git diff + .Minimal + .Prune, .Compact, and .Indent_Heuristic git diff --minimal + .Histogram + .Compact and .Indent_Heuristic git diff --histogram + +Without `.Indent_Heuristic`, each matches git with `--no-indent-heuristic`. + + +## Memory + +Nothing allocates behind your back. There are two ways in: + +- The `*_buffer` procedures (`diff_buffer`, `diff_by_buffer`, `diff_interned_buffer`, and `diff_text_buffer`) + never allocate. Pass `Changes` made from `changes_size` words, and scratch of `scratch_size`, + `scratch_size_interned`, or `scratch_size_text` bytes at any alignment. +- The allocating procedures (`diff`, `diff_by`, `diff_interned`, and `diff_text`) take two allocators with no + defaults. `allocator` gives the result for you to free. `transient_allocator` gives scratch that is freed + before they return. + +The result is allocated first and the scratch last. The scratch is freed first. Heap allocators never leak. +An allocator that frees in reverse order can be passed as both. A bump arena cannot free the scratch: it +stays until the arena is reset. Use separate arenas for the two lifetimes, or a `*_buffer` procedure with +scratch from a temporary region (see the examples). + +Writers take `scratch: []byte`, not an allocator. Only `.Words` emphasis needs scratch: +`emphasis_scratch_size` bytes. + +`unified` and `write_unified_to_builder` make room for their output by `Sizing`: + + .Exact The default. Writes the diff twice and allocates exactly once. Suits any allocator. + .Grow Writes the diff once into growing memory. Close to twice as fast. Suits the heap and allocators + that resize their last allocation in place (e.g. `core:mem`'s and `core:mem/virtual`'s `Arena`). + + +## Limits + +`a` and `b` together must have fewer than `max(i32)` tokens. Each text must be shorter than `max(i32)` bytes. +Going past either is an assertion failure. + + +## Examples + +Example: + package main + + import "core:diff" + import "core:os" + import "core:terminal" + + main :: proc() { + old := "one\ntwo\nthree\n" + new := "one\n2\nthree\nfour\n" + + d, err := diff.diff_text(old, new, context.allocator, context.allocator) + if err != nil { + return + } + defer diff.text_diff_destroy(d, context.allocator) + + style := diff.STYLE_NONE + if terminal.color_enabled { + style = diff.STYLE_ANSI + } + _ = diff.write_unified(os.to_stream(os.stdout), d, "old.txt", "new.txt", style = style, emphasis = .Prefix_Suffix) + } + +Output: + --- old.txt + +++ new.txt + @@ -1,3 +1,4 @@ + one + -two + +2 + three + +four + + +Example: + // Any comparable type, read back as edits + a := []int{1, 2, 3, 4, 5} + b := []int{1, 3, 4, 6, 5} + + c, err := diff.diff(a, b, context.allocator, context.allocator, opts = {algorithm = .Histogram}) + if err != nil { + return + } + defer diff.changes_destroy(c, context.allocator) + + it := diff.edit_iterator_make(c) + for e in diff.edit_iterate(&it) { + switch e.op { + case .Equal: fmt.println("keep ", a[e.a_lo:e.a_hi]) + case .Delete: fmt.println("delete", a[e.a_lo:e.a_hi]) + case .Insert: fmt.println("insert", b[e.b_lo:e.b_hi]) + } + } + +Output: + keep [1] + delete [2] + keep [3, 4] + insert [6] + keep [5] + + +Example: + // Your own equality: equal tokens must hash the same + equal_fold :: proc(x, y: string, _: rawptr) -> bool { + return strings.equal_fold(x, y) + } + hash_fold :: proc(x: string, _: rawptr) -> u64 { + h := u64(0xcbf29ce484222325) + for r in x { + h = (h ~ u64(unicode.to_lower(r))) * 0x100000001b3 + } + return h + } + + a := []string{"Hello", "World"} + b := []string{"hello", "there", "world"} + c, err := diff.diff_by(a, b, equal_fold, hash_fold, nil, context.allocator, context.allocator) + if err != nil { + return + } + defer diff.changes_destroy(c, context.allocator) + fmt.println(diff.stats(c)) + +Output: + Stats{equal = 2, deleted = 0, inserted = 1} + + +Example: + // Words, rendered inline + d, err := diff.diff_text("the quick brown fox", "the slow brown fox", context.allocator, context.allocator, token = .Word) + if err != nil { + return + } + defer diff.text_diff_destroy(d, context.allocator) + + it := diff.span_iterator_make(d) + for s in diff.span_iterate(&it) { + switch s.op { + case .Equal: fmt.print(s.text) + case .Delete: fmt.printf("[-%s-]", s.text) + case .Insert: fmt.printf("{{+%s+}}", s.text) + } + } + +Output: + the [-quick-]{+slow+} brown fox + + +Example: + // One arena: the result first, then the scratch in a temporary region + arena: vmem.Arena + _ = vmem.arena_init_growing(&arena) + defer vmem.arena_destroy(&arena) + arena_allocator := vmem.arena_allocator(&arena) + + n := diff.token_count(old, .Line) + m := diff.token_count(new, .Line) + bits, _ := make([]u64, diff.changes_size(n, m), arena_allocator) + + temp := vmem.arena_temp_begin(&arena) + scratch, _ := make([]byte, diff.scratch_size_text(n, m), arena_allocator) + d := diff.diff_text_buffer(old, new, diff.changes_from_buffer(n, m, bits), scratch) + vmem.arena_temp_end(temp) + + // `d` stays valid until the arena is destroyed +*/ +package diff diff --git a/core/diff/edit.odin b/core/diff/edit.odin new file mode 100644 index 000000000..6055c6376 --- /dev/null +++ b/core/diff/edit.odin @@ -0,0 +1,191 @@ +package diff + +import "base:runtime" + +DEFAULT_CONTEXT_LEN :: 3 + +// What happened to a run of tokens. +Op :: enum u8 { + Equal, + Delete, + Insert, +} + +/* + A run of tokens with the same `Op`: the token ranges `a[a_lo:a_hi]` and `b[b_lo:b_hi]`. + A `.Delete` has an empty `b` range. An `.Insert` has an empty `a` range. +*/ +Edit :: struct { + op: Op, + a_lo, a_hi: int, + b_lo, b_hi: int, +} + +// The token ranges of a group of changes and up to `context_len` equal tokens around it. +Hunk :: struct { + a_lo, a_hi: int, + b_lo, b_hi: int, +} + +Edit_Iterator :: struct { + changes: Changes, + i, j: int, // the next token on each side + a_hi, b_hi: int, // where iteration stops on each side +} + +Hunk_Iterator :: struct { + changes: Changes, + context_len: int, + i, j: int, // the next token on each side +} + + +// Makes an iterator over every edit in `c`. +@(require_results) +edit_iterator_make :: proc "contextless" (c: Changes) -> Edit_Iterator { + return {changes = c, a_hi = c.n, b_hi = c.m} +} + +// Makes an iterator over the edits inside `h`, including its context. +@(require_results) +edit_iterator_make_hunk :: proc "contextless" (c: Changes, h: Hunk) -> Edit_Iterator { + return {changes = c, i = h.a_lo, j = h.b_lo, a_hi = h.a_hi, b_hi = h.b_hi} +} + +/* + Iterates over edits in order. Never allocates. + + Usage: + it := diff.edit_iterator_make(changes) + for e in diff.edit_iterate(&it) { + ... + } +*/ +@(require_results) +edit_iterate :: proc "contextless" (it: ^Edit_Iterator) -> (e: Edit, ok: bool) { + c := it.changes + i, j := it.i, it.j + + switch { + case i < it.a_hi && is_deleted(c, i): + e = {.Delete, i, i + _deleted_run(c, i, it.a_hi), j, j} + case j < it.b_hi && is_inserted(c, j): + e = {.Insert, i, i, j, j + _inserted_run(c, j, it.b_hi)} + case i < it.a_hi && j < it.b_hi: + n := _equal_run(c, i, j, it.a_hi, it.b_hi) + e = {.Equal, i, i + n, j, j + n} + case i < it.a_hi || j < it.b_hi: + panic_contextless("changes do not describe a diff: kept tokens are left over on one side") + case: + return + } + it.i, it.j = e.a_hi, e.b_hi + return e, true +} + +@(require_results) +edit_count :: proc "contextless" (c: Changes) -> (count: int) { + it := edit_iterator_make(c) + for _ in edit_iterate(&it) { + count += 1 + } + return +} + +// Fills `buf` with every edit in `c` and returns the filled part. `buf` must hold at least `edit_count(c)` edits. +@(require_results) +edits_fill :: proc(c: Changes, buf: []Edit, loc := #caller_location) -> []Edit { + n := 0 + it := edit_iterator_make(c) + for e in edit_iterate(&it) { + assert(n < len(buf), "buffer too small for the edits", loc) + buf[n] = e + n += 1 + } + return buf[:n] +} + +// Allocates exactly `edit_count(c)` edits and fills them. +@(require_results) +edits_make :: proc(c: Changes, allocator: runtime.Allocator, loc := #caller_location) -> (edits: []Edit, err: runtime.Allocator_Error) { + edits = make([]Edit, edit_count(c), allocator, loc) or_return + return edits_fill(c, edits, loc), nil +} + + +// A negative `context_len` is treated as 0. +@(require_results) +hunk_iterator_make :: proc "contextless" (c: Changes, context_len := DEFAULT_CONTEXT_LEN) -> Hunk_Iterator { + return {changes = c, context_len = max(context_len, 0)} +} + +/* + Iterates over hunks in order. Hunks whose context would overlap or touch are merged. Never allocates. + + Usage: + hunks := diff.hunk_iterator_make(changes) + for h in diff.hunk_iterate(&hunks) { + edits := diff.edit_iterator_make_hunk(changes, h) + for e in diff.edit_iterate(&edits) { + ... + } + } +*/ +@(require_results) +hunk_iterate :: proc "contextless" (it: ^Hunk_Iterator) -> (h: Hunk, ok: bool) { + c := it.changes + ctx := it.context_len + + // Skip to the next change + gap := _equal_run(c, it.i, it.j, c.n, c.m) + i, j := it.i + gap, it.j + gap + if i == c.n && j == c.m { + return + } + lead := min(ctx, gap) + h.a_lo, h.b_lo = i - lead, j - lead + + for { + // Take the change: its deletions, then its insertions + deleted := _deleted_run (c, i, c.n) + inserted := _inserted_run(c, j, c.m) + if deleted == 0 && inserted == 0 { + panic_contextless("changes do not describe a diff: kept tokens are left over on one side") + } + i += deleted + j += inserted + + gap = _equal_run(c, i, j, c.n, c.m) + if gap - ctx > ctx || (i + gap == c.n && j + gap == c.m) { + trail := min(ctx, gap) + h.a_hi, h.b_hi = i + trail, j + trail + it.i, it.j = i, j + return h, true + } + + // Close enough to the next change for their context to touch. They share a hunk + i += gap + j += gap + } +} + + +// Returns how many equal pairs there are from `a[i]` and `b[j]`, up to `a_hi` and `b_hi`. +@(require_results) +_equal_run :: proc "contextless" (c: Changes, i, j, a_hi, b_hi: int) -> int { + offset := _inserted_offset(c) + return min(_bit_run(c.bits, i, a_hi, false), _bit_run(c.bits, offset + j, offset + b_hi, false)) +} + +// Returns how many tokens from `a[i]`, up to `a_hi`, are deleted. +@(require_results) +_deleted_run :: proc "contextless" (c: Changes, i, a_hi: int) -> int { + return _bit_run(c.bits, i, a_hi, true) +} + +// Returns how many tokens from `b[j]`, up to `b_hi`, are inserted. +@(require_results) +_inserted_run :: proc "contextless" (c: Changes, j, b_hi: int) -> int { + offset := _inserted_offset(c) + return _bit_run(c.bits, offset + j, offset + b_hi, true) +} diff --git a/core/diff/emphasis.odin b/core/diff/emphasis.odin new file mode 100644 index 000000000..f93b9d1dc --- /dev/null +++ b/core/diff/emphasis.odin @@ -0,0 +1,304 @@ +package diff + +import "core:io" +import "core:unicode" +import "core:unicode/utf8" + +/* + Emphasis highlights the changed part of changed lines. It works on blocks: the deleted lines of a change + and the inserted lines after them. A block with only one side has nothing to compare. + + - `.Prefix_Suffix` pairs the lines of a block in order when both sides have as many. It highlights what + lies between their common prefix and suffix. + - `.Words` diffs the words of the deleted text against those of the inserted text. It highlights the + changed words, even across lines. + + Parts sharing nothing but whitespace stay plain. Highlighting all of them would say nothing. +*/ + +// One `.Delete` edit and the `.Insert` edit straight after it, being written. +@(private) +Emphasis_Block :: struct { + active: bool, + emphasis: Emphasis, + del, ins: string, // the block's deleted and inserted lines, newlines included + + // Byte offset in `del` and `ins` of the next line to be written + del_line, ins_line: int, + + // `.Prefix_Suffix`: byte offset of the next line's partner on the other side + del_partner, ins_partner: int, + + // `.Words`: the word diff of `del` against `ins`, and the next word on each side + words: Changes, + del_word, ins_word: Word_Cursor, +} + +@(private) +Word_Cursor :: struct { + pos: int, // byte offset of the word + index: int, // which word it is +} + +// Returns the scratch bytes `write_unified`, `unified_size`, and `unified` need for `emphasis`. 0 unless it is `.Words`. +@(require_results) +emphasis_scratch_size :: proc(d: Text_Diff, emphasis: Emphasis, loc := #caller_location) -> int { + if emphasis != .Words { + return 0 + } + assert(d.token == .Line, "emphasis needs a line diff", loc) + + largest := 0 + a_pos, b_pos := 0, 0 + del: string + + it := edit_iterator_make(d.changes) + for e in edit_iterate(&it) { + a_end := _skip_tokens(d.a, a_pos, e.a_hi - e.a_lo, .Line) + b_end := _skip_tokens(d.b, b_pos, e.b_hi - e.b_lo, .Line) + + switch e.op { + case .Equal: + del = "" + case .Delete: + del = d.a[a_pos:a_end] + case .Insert: + if del != "" { + n, m := token_count(del, .Word), token_count(d.b[b_pos:b_end], .Word) + largest = max(largest, words_scratch_size(n, m)) + } + del = "" + } + + a_pos, b_pos = a_end, b_end + } + + return largest +} + +// The scratch the word diff of a block of `n` and `m` words needs, its `Changes` included. +@(private, require_results) +words_scratch_size :: proc "contextless" (n, m: int) -> int { + return SCRATCH_ALIGN + scratch_bytes(u64, changes_size(n, m)) + scratch_size_text(n, m) +} + +/* + Sets up the block starting at `line`: the first line of the iterator's `.Delete` edit. Leaves it + inactive when there is nothing to compare it with. +*/ +@(private, require_results) +emphasis_block_make :: proc(it: ^Unified_Iterator, line: Unified_Line, emphasis: Emphasis, scratch: []byte, loc := #caller_location) -> (block: Emphasis_Block) { + d := it.d + del := it.edit + + // The iterator has already moved past `line` + line_len := len(line.text) + if !it.no_newline { + line_len += 1 + } + del_start := it.a_pos - line_len + del_end := _skip_tokens(d.a, del_start, del.a_hi - del.a_lo, .Line) + + peek := it.edits + ins, has_ins := edit_iterate(&peek) + if !has_ins || ins.op != .Insert { + return + } + ins_end := _skip_tokens(d.b, it.b_pos, ins.b_hi - ins.b_lo, .Line) + + block.emphasis = emphasis + block.del = d.a[del_start:del_end] + block.ins = d.b[it.b_pos:ins_end] + + switch emphasis { + case .None: + return + + case .Prefix_Suffix: + block.active = del.a_hi - del.a_lo == ins.b_hi - ins.b_lo + + case .Words: + n, m := token_count(block.del, .Word), token_count(block.ins, .Word) + assert(len(scratch) >= words_scratch_size(n, m), ".Words emphasis needs `emphasis_scratch_size` bytes of scratch", loc) + + s := Scratch{buf = scratch} + bits := scratch_take(&s, u64, changes_size(n, m), loc) + block.words = changes_from_buffer(n, m, bits, loc) + + words := diff_text_buffer(block.del, block.ins, block.words, scratch[s.pos:], .Word, loc = loc) + + @(require_results) + is_all_space :: proc(s: string) -> bool { + for r in s { + if !unicode.is_space(r) { + return false + } + } + return true + } + + // Sharing only whitespace, such as the line breaks, is not having anything in common + spans := span_iterator_make(words) + for span in span_iterate(&spans) { + if span.op == .Equal && !is_all_space(span.text) { + block.active = true + break + } + } + } + return +} + +// Writes a deleted or inserted line of `block`, highlighting its changed part. +@(private) +write_emphasized :: proc(w: io.Writer, block: ^Emphasis_Block, kind: Unified_Kind, text: string, style: Style) -> io.Error { + Pen :: struct { + w: io.Writer, + base: string, + emph: string, + reset: string, + emphasized: bool, + colored: bool, // a colour is on: `reset` is due before the next one or at the end + } + + pen_start :: proc(p: ^Pen) -> io.Error { + if p.base != "" { + io.write_string(p.w, p.base) or_return + p.colored = true + } + return nil + } + + pen_write :: proc(p: ^Pen, text: string, emphasized: bool) -> io.Error { + if text == "" { + return nil + } + + switch { + case emphasized && !p.emphasized: + if p.emph != "" { + io.write_string(p.w, p.emph) or_return + p.colored = true + } + case !emphasized && p.emphasized: + if p.colored { + io.write_string(p.w, p.reset) or_return + p.colored = false + } + pen_start(p) or_return + } + p.emphasized = emphasized + + _, err := io.write_string(p.w, text) + return err + } + + pen_end :: proc(p: ^Pen) -> io.Error { + if p.colored { + _, err := io.write_string(p.w, p.reset) + return err + } + return nil + } + + // The line at `s[pos]` without its "\n", and where the next line starts + @(require_results) + line_at :: proc(s: string, pos: int) -> (text: string, next: int) { + next = _token_end(s, pos, .Line) + text = s[pos:next] + if len(text) > 0 && text[len(text)-1] == '\n' { + text = text[:len(text)-1] + } + return + } + + base, emph, prefix := style.delete, style.delete_emphasis, "-" + if kind == .Insert { + base, emph, prefix = style.insert, style.insert_emphasis, "+" + } + pen := Pen{w = w, base = base, emph = emph, reset = style.reset} + pen_start(&pen) or_return + io.write_string(w, prefix) or_return + + switch block.emphasis { + case .None: + io.write_string(w, text) or_return + + case .Prefix_Suffix: + partner: string + if kind == .Delete { + partner, block.ins_partner = line_at(block.ins, block.ins_partner) + } else { + partner, block.del_partner = line_at(block.del, block.del_partner) + } + + prefix_len, suffix_len := _common_affixes(text, partner) + if prefix_len == 0 && suffix_len == 0 { + io.write_string(w, text) or_return + } else { + pen_write(&pen, text[:prefix_len], false) or_return + pen_write(&pen, text[prefix_len:len(text)-suffix_len], true) or_return + pen_write(&pen, text[len(text)-suffix_len:], false) or_return + } + + case .Words: + src, line_pos, cursor := block.del, block.del_line, &block.del_word + if kind == .Insert { + src, line_pos, cursor = block.ins, block.ins_line, &block.ins_word + } + line_end := line_pos + len(text) + + for cursor.pos < line_end { + word_end := _token_end(src, cursor.pos, .Word) + changed := is_deleted(block.words, cursor.index) + if kind == .Insert { + changed = is_inserted(block.words, cursor.index) + } + + // A word of whitespace can run on past the end of the line, into the next + lo, hi := max(cursor.pos, line_pos), min(word_end, line_end) + pen_write(&pen, src[lo:hi], changed) or_return + if word_end > line_end { + break + } + cursor.pos = word_end + cursor.index += 1 + } + } + + if kind == .Delete { + _, block.del_line = line_at(block.del, block.del_line) + } else { + _, block.ins_line = line_at(block.ins, block.ins_line) + } + pen_end(&pen) or_return + return io.write_byte(w, '\n') +} + + +// Returns how many bytes `x` and `y` share at the start and at the end, on rune boundaries and without overlapping. +@(require_results) +_common_affixes :: proc(x, y: string) -> (prefix, suffix: int) { + // Whether a rune starts at `s[i]`, counting the end of `s` as a start + @(require_results) + rune_starts :: #force_inline proc "contextless" (s: string, i: int) -> bool { + return i >= len(s) || utf8.rune_start(s[i]) + } + + + n := min(len(x), len(y)) + for prefix < n && x[prefix] == y[prefix] { + prefix += 1 + } + for prefix > 0 && !(rune_starts(x, prefix) && rune_starts(y, prefix)) { + prefix -= 1 + } + + for suffix < n - prefix && x[len(x)-1-suffix] == y[len(y)-1-suffix] { + suffix += 1 + } + for suffix > 0 && !(rune_starts(x, len(x)-suffix) && rune_starts(y, len(y)-suffix)) { + suffix -= 1 + } + return +} diff --git a/core/diff/histogram.odin b/core/diff/histogram.odin new file mode 100644 index 000000000..735cb1129 --- /dev/null +++ b/core/diff/histogram.odin @@ -0,0 +1,191 @@ +#+private +package diff + +/* + Histogram diff. Each region is split around a run of equal tokens anchored on its rarest tokens. The regions + before and after the run are then diffed the same way. How rare a token is counts only the region of `a`. + + A token occurring more than `HISTOGRAM_MAX_OCCURRENCES` times in the region cannot anchor a run. A region whose + tokens in common are all that frequent is diffed again by `rediff`: Myers after `.Prune`'s preparation. +*/ + +// A token occurring more often than this in the region of `a` is too frequent to anchor on. +HISTOGRAM_MAX_OCCURRENCES :: 64 + +// The end of a chain of occurrences. +HISTOGRAM_NONE :: -1 + +Histogram :: struct { + a, b: []i32, + count: []i32, // the occurrences of each ID in the region of `a`, zero outside a search + first: []i32, // the first occurrence of each ID in the region of `a`, wherever `count` is not 0 + next: []i32, // the next occurrence of `a[i]` in the region, or `HISTOGRAM_NONE` +} + +// What the search of a region found. +Histogram_Found :: enum u8 { + Nothing, // no tokens in common + Run, // a run to split around + Fall_Back, // only tokens too frequent to anchor on +} + +@(require_results) +histogram_scratch_size :: proc "contextless" (n, m: int, id_count: int) -> int { + return 2*scratch_bytes(i32, id_count) + scratch_bytes(i32, n) + rediff_scratch_size(n, m, id_count) +} + +// Diffs `a` against `b` into zeroed `changes`. Gives back its scratch. +histogram :: proc(a, b: []i32, id_count: int, changes: Changes, s: ^Scratch, max_cost: int, loc := #caller_location) { + start := s.pos + h := Histogram{ + a = a, + b = b, + count = scratch_take(s, i32, id_count, loc), + first = scratch_take(s, i32, id_count, loc), + next = scratch_take(s, i32, len(a), loc), + } + for &x in h.count { + x = 0 + } + + stack: [MYERS_MAX_DEPTH]Box + depth := 0 + box := Box{0, len(a), 0, len(b), false} + for { + switch { + case box.a_lo == box.a_hi: + _mark_inserted(changes, box.b_lo, box.b_hi) + + case box.b_lo == box.b_hi: + _mark_deleted(changes, box.a_lo, box.a_hi) + + case: + run, found := histogram_best_run(&h, box) + switch found { + case .Fall_Back: + this := Compact_Side{a, 0, nil} + other := Compact_Side{b, _inserted_offset(changes), nil} + rediff(changes, this, other, {box.a_lo, box.a_hi}, {box.b_lo, box.b_hi}, id_count, max_cost, s, loc) + + case .Nothing: + _mark_deleted(changes, box.a_lo, box.a_hi) + _mark_inserted(changes, box.b_lo, box.b_hi) + + case .Run: + lo := Box{box.a_lo, run.a_lo, box.b_lo, run.b_lo, false} + hi := Box{run.a_hi, box.a_hi, run.b_hi, box.b_hi, false} + + // The smaller side first, keeping the stack within `MYERS_MAX_DEPTH` + if box_size(lo) <= box_size(hi) { + stack[depth] = hi + box = lo + } else { + stack[depth] = lo + box = hi + } + depth += 1 + continue + } + } + + if depth == 0 { + break + } + depth -= 1 + box = stack[depth] + } + s.pos = start +} + +/* + Finds the run of equal tokens to split `box` around. Both sides are non-empty. + + Each token of `b`, in order, anchors a run at every occurrence of its ID in `a` when it is rare enough. The + run grows both ways. It is the best if it is longer than the best so far, or if its rarest token is rarer. + Tokens of `b` inside a run are not tried on their own. +*/ +@(require_results) +histogram_best_run :: proc(h: ^Histogram, box: Box) -> (run: Box, found: Histogram_Found) { + a, b := h.a, h.b + count := h.count + first := h.first + next := h.next + + // Chain the occurrences of each ID in the region of `a`, from the lowest index + for i := box.a_hi - 1; i >= box.a_lo; i -= 1 { + id := a[i] + next[i] = HISTOGRAM_NONE + if count[id] > 0 { + next[i] = first[id] + } + first[id] = i32(i) + count[id] += 1 + } + + best_count := HISTOGRAM_MAX_OCCURRENCES + 1 + best_span := 0 // the best run's length in `a` minus one + has_common := false + has_run := false + for j := box.b_lo; j < box.b_hi; /**/ { + next_j := j + 1 + c := int(count[b[j]]) + switch { + case c == 0: + // Not in the region of `a` + case c > best_count: + has_common = true + case: + has_common = true + for i := int(first[b[j]]); i != HISTOGRAM_NONE; /**/ { + // Grow the run around `a[i] == b[j]`, noting its rarest token + rarest := c + a_lo, b_lo := i, j + for a_lo > box.a_lo && b_lo > box.b_lo && a[a_lo-1] == b[b_lo-1] { + a_lo -= 1 + b_lo -= 1 + if rarest > 1 { + rarest = min(rarest, int(count[a[a_lo]])) + } + } + a_last, b_last := i, j + for a_last < box.a_hi - 1 && b_last < box.b_hi - 1 && a[a_last+1] == b[b_last+1] { + a_last += 1 + b_last += 1 + if rarest > 1 { + rarest = min(rarest, int(count[a[a_last]])) + } + } + + next_j = max(next_j, b_last + 1) + if a_last - a_lo > best_span || rarest < best_count { + run = {a_lo, a_last + 1, b_lo, b_last + 1, false} + best_span = a_last - a_lo + best_count = rarest + has_run = true + } + + // The next occurrence past the run + i = int(next[i]) + for i != HISTOGRAM_NONE && i <= a_last { + i = int(next[i]) + } + } + } + j = next_j + } + + // Leave the counts zeroed for the next region + for i in box.a_lo.. HISTOGRAM_MAX_OCCURRENCES: + found = .Fall_Back + case has_run: + found = .Run + case: + found = .Nothing + } + return +} diff --git a/core/diff/intern.odin b/core/diff/intern.odin new file mode 100644 index 000000000..09632e8af --- /dev/null +++ b/core/diff/intern.odin @@ -0,0 +1,125 @@ +#+private +package diff + +import "base:intrinsics" + +/* + Interning gives every token of `a`, then `b`, a dense ID in order of first appearance. The algorithms then + only ever compare `i32`s. An open-addressing hash table in the scratch buffer finds equal tokens. + The algorithm reuses that space once interning is done. +*/ + +// An empty slot has `token == 0`. Otherwise `token - 1` indexes the first token seen with this value. +Intern_Slot :: struct { + token: i32, + hash: u32, +} + +// The table is a power of two, at least twice the number of tokens. It stays at most half full. +@(require_results) +intern_table_len :: proc "contextless" (tokens: int) -> int { + size := 2 + for size < 2*tokens { + size <<= 1 + } + return size +} + +@(require_results) +intern_scratch_size :: proc "contextless" (tokens: int) -> int { + return scratch_bytes(Intern_Slot, intern_table_len(tokens)) +} + +// One string's tokens, without a slice of them. `starts` holds where each token starts, then the string's length. +Text_Tokens :: struct { + text: string, + starts: []i32, +} + +@(require_results) +slice_tokens_len :: #force_inline proc "contextless" (s: []$T) -> int { + return len(s) +} + +@(require_results) +text_tokens_len :: #force_inline proc "contextless" (s: Text_Tokens) -> int { + return len(s.starts) - 1 +} + +@(require_results) +slice_token_at :: #force_inline proc "contextless" (s: []$T, k: int) -> T { + return s[k] +} + +@(require_results) +text_token_at :: #force_inline proc "contextless" (s: Text_Tokens, k: int) -> string { + return s.text[s.starts[k]:s.starts[k+1]] +} + +tokens_len :: proc{ + slice_tokens_len, + text_tokens_len, +} + +token_at :: proc{ + slice_token_at, + text_token_at, +} + +// Fills `ids` with the ID of each token of `a`, then of `b`, and returns how many distinct IDs there are. +@(require_results) +intern :: proc( + a, b: $S, // `[]T` or `Text_Tokens` + $T: typeid, + equal: proc(x, y: T, user_data: rawptr) -> bool, + hash: proc(x: T, user_data: rawptr) -> u64, + user_data: rawptr, + ids: []i32, + table: []Intern_Slot, +) -> (id_count: int) { + // Token `k` of `a`, then of `b`, where `a` has `n` + @(require_results) + token :: #force_inline proc "contextless" (a, b: S, n, k: int) -> T { + if k < n { + return token_at(a, k) + } + return token_at(b, k - n) + } + + + for &slot in table { + slot = {} + } + + // Fibonacci hashing: the top bits of the product pick the slot + shift := 64 - uint(intrinsics.count_trailing_zeros(u64(len(table)))) + mask := len(table) - 1 + + n := tokens_len(a) + for k in 0..> shift) + tag := u32(h) + + for { + slot := &table[pos] + if slot.token == 0 { + slot^ = {token = i32(k + 1), hash = tag} + ids[k] = i32(id_count) + id_count += 1 + break + } + + if slot.hash == tag { + first := int(slot.token - 1) + if equal(x, token(a, b, n, first), user_data) { + ids[k] = ids[first] + break + } + } + pos = (pos + 1) & mask + } + } + return +} diff --git a/core/diff/myers.odin b/core/diff/myers.odin new file mode 100644 index 000000000..27917a08e --- /dev/null +++ b/core/diff/myers.odin @@ -0,0 +1,383 @@ +#+private +package diff + +/* + Myers' O(ND) diff in linear space: Eugene W. Myers, "An O(ND) Difference Algorithm and Its Variations" + (Algorithmica, 1986), section 4b. + + Each box is trimmed of its equal ends, then split where a search forward from its top-left corner meets a + search backward from its bottom-right corner. Both halves are diffed the same way. Positions are `(x, y)`, + with `x` indexing `a` and `y` indexing `b`, on diagonal `k = x - y`. + + A box that is not `minimal` can give up on an exact split in two ways: + - the long-snake heuristic: past `MYERS_HEURISTIC_COST` edits, at a long run of equal pairs that has come far + - the cost cap: at `max_cost` edits, at the furthest either search has come + + The half the search has explored then becomes minimal, and the other half does not. An exact split makes + both halves minimal. +*/ + +// The depth of the work stacks. Taking the smaller half of each split first keeps them within it. +MYERS_MAX_DEPTH :: 64 + +// The default cost cap is never lower than this. +MYERS_MIN_COST :: 256 + +// The long-snake heuristic only starts past this many edits. +MYERS_HEURISTIC_COST :: 256 + +// A snake is long past this many equal pairs. A split by the heuristic needs this many equal pairs beside it. +MYERS_SNAKE_LEN :: 20 + +// A split by the heuristic must have come further than this many times the edits. +MYERS_HEURISTIC_FACTOR :: 4 + +// What a search holds on a diagonal it has not reached yet. +MYERS_FORWARD_UNREACHED :: -1 +MYERS_BACKWARD_UNREACHED :: max(i32) + +// `a[a_lo:a_hi]` against `b[b_lo:b_hi]`. A minimal box is always diffed exactly. +Box :: struct { + a_lo, a_hi: int, + b_lo, b_hi: int, + minimal: bool, +} + +/* + Diffs any box within `a` and `b`. For each diagonal, `fwd` holds the largest `x` the forward search has + reached, and `bwd` the smallest `x` the backward search has reached. Both are indexed by the diagonal + itself, from `-len(b)-1` to `len(a)+1`. +*/ +Myers :: struct { + a, b: []i32, + fwd, bwd: [^]i32, +} + +@(require_results) +box_size :: proc "contextless" (box: Box) -> int { + return (box.a_hi - box.a_lo) + (box.b_hi - box.b_lo) +} + +// The scratch `myers_make` takes for inputs of length `n` and `m`. +@(require_results) +myers_scratch_size :: proc "contextless" (n, m: int) -> int { + return 2*scratch_bytes(i32, n + m + 3) +} + +// The cost cap for `n` by `m` tokens: `max_cost` when it is set, otherwise about the square root of the tokens. +@(require_results) +myers_cost :: proc "contextless" (max_cost, n, m: int) -> int { + if max_cost > 0 { + return max_cost + } + return max(rough_sqrt(n + m + 3), MYERS_MIN_COST) +} + +// A power of two near the square root of `n`: one bit for every two bits of `n`. 1 for 0. +@(require_results) +rough_sqrt :: proc "contextless" (n: int) -> (r: int) { + r = 1 + for rest := n; rest > 0; rest >>= 2 { + r <<= 1 + } + return +} + +// Diffs `a` against `b` into zeroed `changes`. Gives back its scratch. +myers :: proc(a, b: []i32, changes: Changes, s: ^Scratch, minimal: bool, max_cost: int, loc := #caller_location) { + start := s.pos + md := myers_make(a, b, s, loc) + myers_box(&md, changes, {0, len(a), 0, len(b), minimal}, myers_cost(max_cost, len(a), len(b)), loc) + s.pos = start +} + +// Takes `myers_scratch_size(len(a), len(b))` bytes from `s`. +@(require_results) +myers_make :: proc(a, b: []i32, s: ^Scratch, loc := #caller_location) -> Myers { + diagonals := len(a) + len(b) + 3 + fwd := scratch_take(s, i32, diagonals, loc) + bwd := scratch_take(s, i32, diagonals, loc) + + // Diagonal 0 sits after the `len(b) + 1` diagonals below it + zero := len(b) + 1 + return { + a = a, + b = b, + fwd = raw_data(fwd)[zero:], + bwd = raw_data(bwd)[zero:], + } +} + +// Diffs `box` into `changes`, marking bits only inside it. Boxes that are not minimal give up at `max_cost` edits. +myers_box :: proc(md: ^Myers, changes: Changes, box: Box, max_cost: int, loc := #caller_location) { + a, b := md.a, md.b + assert(0 <= box.a_lo && box.a_lo <= box.a_hi && box.a_hi <= len(a), "box outside the inputs", loc) + assert(0 <= box.b_lo && box.b_lo <= box.b_hi && box.b_hi <= len(b), "box outside the inputs", loc) + + stack: [MYERS_MAX_DEPTH]Box + depth := 0 + box := box + for { + // Equal ends need nothing marked + for box.a_lo < box.a_hi && box.b_lo < box.b_hi && a[box.a_lo] == b[box.b_lo] { + box.a_lo += 1 + box.b_lo += 1 + } + for box.a_lo < box.a_hi && box.b_lo < box.b_hi && a[box.a_hi-1] == b[box.b_hi-1] { + box.a_hi -= 1 + box.b_hi -= 1 + } + + switch { + case box.a_lo == box.a_hi: + _mark_inserted(changes, box.b_lo, box.b_hi) + + case box.b_lo == box.b_hi: + _mark_deleted(changes, box.a_lo, box.a_hi) + + case: + lo, hi := myers_split(md, box, max_cost) + assert(box.a_lo <= lo.a_hi && lo.a_hi <= box.a_hi, "Myers split outside its box", loc) + assert(box.b_lo <= lo.b_hi && lo.b_hi <= box.b_hi, "Myers split outside its box", loc) + + // A half is only the whole box when it is handed on as minimal + assert(box_size(lo) < box_size(box) || (lo.minimal && !box.minimal), "Myers split made no progress", loc) + assert(box_size(hi) < box_size(box) || (hi.minimal && !box.minimal), "Myers split made no progress", loc) + + // The smaller half first, keeping the stack within `MYERS_MAX_DEPTH` + if box_size(lo) <= box_size(hi) { + stack[depth] = hi + box = lo + } else { + stack[depth] = lo + box = hi + } + depth += 1 + continue + } + + if depth == 0 { + break + } + depth -= 1 + box = stack[depth] + } +} + +/* + Splits `box` in two where the forward and backward searches meet, or where a box that is not minimal gives + up. Both sides are non-empty, and the first and last tokens differ. +*/ +@(require_results) +myers_split :: proc(md: ^Myers, box: Box, max_cost: int) -> (lo, hi: Box) { + @(require_results) + split_at :: #force_inline proc "contextless" (box: Box, x, y: int, lo_minimal, hi_minimal: bool) -> (lo, hi: Box) { + lo = {box.a_lo, x, box.b_lo, y, lo_minimal} + hi = {x, box.a_hi, y, box.b_hi, hi_minimal} + return + } + + // Whether `MYERS_SNAKE_LEN` pairs from `a[x]` and `b[y]` on are equal + @(require_results) + equal_run :: proc "contextless" (a, b: []i32, x, y: int) -> bool { + for t in 0.. diag_lo { + fwd_lo -= 1 + fwd[fwd_lo-1] = MYERS_FORWARD_UNREACHED + } else { + fwd_lo += 1 + } + if fwd_hi < diag_hi { + fwd_hi += 1 + fwd[fwd_hi+1] = MYERS_FORWARD_UNREACHED + } else { + fwd_hi -= 1 + } + + for k := fwd_hi; k >= fwd_lo; k -= 2 { + // From the diagonal below by a deletion, ties included, or from the one above by an insertion + x: int + if fwd[k-1] >= fwd[k+1] { + x = int(fwd[k-1]) + 1 + } else { + x = int(fwd[k+1]) + } + y := x - k + + start := x + for x < a_hi && y < b_hi && a[x] == b[y] { + x += 1 + y += 1 + } + if x - start > MYERS_SNAKE_LEN { + long_snake = true + } + + fwd[k] = i32(x) + if meet_forward && bwd_lo <= k && k <= bwd_hi && int(bwd[k]) <= x { + return split_at(box, x, y, true, true) + } + } + + // Backward: the same from the other corner + if bwd_lo > diag_lo { + bwd_lo -= 1 + bwd[bwd_lo-1] = MYERS_BACKWARD_UNREACHED + } else { + bwd_lo += 1 + } + if bwd_hi < diag_hi { + bwd_hi += 1 + bwd[bwd_hi+1] = MYERS_BACKWARD_UNREACHED + } else { + bwd_hi -= 1 + } + + for k := bwd_hi; k >= bwd_lo; k -= 2 { + // From the diagonal below by an insertion, or from the one above by a deletion, ties included + x: int + if bwd[k-1] < bwd[k+1] { + x = int(bwd[k-1]) + } else { + x = int(bwd[k+1]) - 1 + } + y := x - k + + end := x + for x > a_lo && y > b_lo && a[x-1] == b[y-1] { + x -= 1 + y -= 1 + } + if end - x > MYERS_SNAKE_LEN { + long_snake = true + } + + bwd[k] = i32(x) + if !meet_forward && fwd_lo <= k && k <= fwd_hi && x <= int(fwd[k]) { + return split_at(box, x, y, true, true) + } + } + + if box.minimal { + continue + } + + // The long-snake heuristic: split after a long snake that has come far, forward first + if long_snake && d > MYERS_HEURISTIC_COST { + best := 0 + best_x := 0 + best_y := 0 + for k := fwd_hi; k >= fwd_lo; k -= 2 { + x := int(fwd[k]) + y := x - k + v := (x - a_lo) + (y - b_lo) - abs(k - fwd_start) + if v > MYERS_HEURISTIC_FACTOR*d && + v > best && + a_lo + MYERS_SNAKE_LEN <= x && + x < a_hi && + b_lo + MYERS_SNAKE_LEN <= y && + y < b_hi && + equal_run(a, b, x - MYERS_SNAKE_LEN, y - MYERS_SNAKE_LEN) { + best = v + best_x = x + best_y = y + } + } + if best > 0 { + return split_at(box, best_x, best_y, true, false) + } + + for k := bwd_hi; k >= bwd_lo; k -= 2 { + x := int(bwd[k]) + y := x - k + v := (a_hi - x) + (b_hi - y) - abs(k - bwd_start) + if v > MYERS_HEURISTIC_FACTOR*d && + v > best && + a_lo < x && + x <= a_hi - MYERS_SNAKE_LEN && + b_lo < y && + y <= b_hi - MYERS_SNAKE_LEN && + equal_run(a, b, x, y) { + best = v + best_x = x + best_y = y + } + } + if best > 0 { + return split_at(box, best_x, best_y, false, true) + } + } + + // The cost cap: split where the search that has come further got to + if d >= max_cost { + fwd_sum := -1 + fwd_x := 0 + fwd_y := 0 + for k := fwd_hi; k >= fwd_lo; k -= 2 { + x := min(int(fwd[k]), a_hi) + y := x - k + if y > b_hi { + x = b_hi + k + y = b_hi + } + if x + y > fwd_sum { + fwd_sum = x + y + fwd_x = x + fwd_y = y + } + } + + bwd_sum := max(int) + bwd_x := 0 + bwd_y := 0 + for k := bwd_hi; k >= bwd_lo; k -= 2 { + x := max(int(bwd[k]), a_lo) + y := x - k + if y < b_lo { + x = b_lo + k + y = b_lo + } + if x + y < bwd_sum { + bwd_sum = x + y + bwd_x = x + bwd_y = y + } + } + + if (a_hi + b_hi) - bwd_sum < fwd_sum - (a_lo + b_lo) { + return split_at(box, fwd_x, fwd_y, true, false) + } + return split_at(box, bwd_x, bwd_y, false, true) + } + } +} diff --git a/core/diff/prune.odin b/core/diff/prune.odin new file mode 100644 index 000000000..579589f53 --- /dev/null +++ b/core/diff/prune.odin @@ -0,0 +1,205 @@ +#+private +package diff + +/* + The `.Prune` pre-pass. It sets aside the common prefix and suffix, the tokens with no match on the other side, + and some of the frequent tokens among those. The algorithm then diffs only the tokens kept, and its result is + mapped back. + + A token is frequent when its ID occurs on the other side about `sqrt(n)` times or more, for this side's length + `n`. No token is frequent with `.Minimal`. A frequent token is set aside only from inside a run of tokens that + are not kept, with tokens that have no match on both sides of it, and only when few of that run are frequent. +*/ + +// The limit for a frequent token is never higher than this. +PRUNE_MAX_MATCHES :: 1024 + +// How far to look back and ahead from a frequent token. +PRUNE_WINDOW :: 100 + +// A frequent token is set aside when fewer than one in this many of the run around it are frequent. +PRUNE_RUN_FACTOR :: 4 + +// What the other side says about a token of a middle. +Prune_Class :: enum u8 { + Discard, // no match: set aside + Keep, // a few matches: kept + Frequent, // many matches: decided by the tokens around it +} + +/* + The kept tokens and their `Changes` come first. The counts and classes come next. The algorithm reuses their + space once they are done. +*/ +@(require_results) +prune_scratch_size :: proc "contextless" (n, m: int, id_count: int, opts: Options) -> int { + kept := scratch_bytes(i32, n) + scratch_bytes(i32, m) + classes := 2*scratch_bytes(i32, id_count) + scratch_bytes(Prune_Class, n) + scratch_bytes(Prune_Class, m) + algorithm := scratch_bytes(u64, changes_size(n, m)) + unpruned_scratch_size(n, m, id_count, opts) + return kept + max(classes, algorithm) +} + +// Diffs `a` against `b` into zeroed `changes`, with the tokens that cannot usefully match set aside. +prune :: proc(a, b: []i32, id_count: int, changes: Changes, s: ^Scratch, opts: Options, loc := #caller_location) { + // Gives each token of `ids` its class, by the occurrences of its ID in the whole of the other side + classify :: proc "contextless" (class: []Prune_Class, ids: []i32, other_count: []i32, limit: int) { + for id, i in ids { + count := int(other_count[id]) + switch { + case count == 0: class[i] = .Discard + case count < limit: class[i] = .Keep + case: class[i] = .Frequent + } + } + } + + /* + Counts the tokens that are not kept, from `class[from]` on by `step`, for `PRUNE_WINDOW` tokens at most. + Stops at the first kept token or at the end of the middle. + */ + @(require_results) + count_run :: proc "contextless" (class: []Prune_Class, from, step: int) -> (discards, frequent: int) { + frequent = 1 + for r, steps := from, 0; 0 <= r && r < len(class) && steps < PRUNE_WINDOW; r, steps = r + step, steps + 1 { + switch class[r] { + case .Keep: return + case .Discard: discards += 1 + case .Frequent: frequent += 1 + } + } + return + } + + // Whether the frequent token at `class[i]` is set aside: discards on both sides, and few frequent tokens + @(require_results) + set_aside :: proc "contextless" (class: []Prune_Class, i: int) -> bool { + back_discards, back_frequent := count_run(class, i - 1, -1) + if back_discards == 0 { + return false + } + ahead_discards, ahead_frequent := count_run(class, i + 1, +1) + if ahead_discards == 0 { + return false + } + + frequent := back_frequent + ahead_frequent + discards := back_discards + ahead_discards + return frequent*PRUNE_RUN_FACTOR < frequent + discards + } + + /* + Copies the kept tokens of a middle to `kept`, and sets the bits of the others from `offset` in `bits`. + Each frequent token is decided from the classes alone. + */ + @(require_results) + keep :: proc "contextless" (kept: []i32, ids: []i32, class: []Prune_Class, bits: []u64, offset: int) -> (count: int) { + for id, i in ids { + if class[i] == .Keep || (class[i] == .Frequent && !set_aside(class, i)) { + kept[count] = id + count += 1 + } else { + _set_bit_range(bits, offset + i, offset + i + 1) + } + } + return + } + + // Copies the shorter diff's bits, from `kept[kept_lo]` on, to the kept tokens of `bits[lo:hi]`: those still clear + map_back :: proc "contextless" (bits: []u64, lo, hi: int, kept: []u64, kept_lo: int) { + i := lo + k := kept_lo + for i < hi { + i += _bit_run(bits, i, hi, true) + + run := _bit_run(bits, i, hi, false) + for t := 0; t < run; /**/ { + t += _bit_run(kept, k + t, k + run, false) + + changed := _bit_run(kept, k + t, k + run, true) + _set_bit_range(bits, i + t, i + t + changed) + + t += changed + } + + i += run + k += run + } + } + + + n, m := len(a), len(b) + + // The common ends are equal: nothing to mark + prefix := 0 + for prefix < n && prefix < m && a[prefix] == b[prefix] { + prefix += 1 + } + suffix := 0 + for suffix < n - prefix && suffix < m - prefix && a[n-1-suffix] == b[m-1-suffix] { + suffix += 1 + } + a_mid := a[prefix:n-suffix] + b_mid := b[prefix:m-suffix] + + // With one middle empty, every token of the other is a change whatever it is classed as + switch { + case len(a_mid) == 0: + _mark_inserted(changes, prefix, m - suffix) + return + case len(b_mid) == 0: + _mark_deleted(changes, prefix, n - suffix) + return + } + + start := s.pos + kept_a := scratch_take(s, i32, len(a_mid), loc) + kept_b := scratch_take(s, i32, len(b_mid), loc) + + // The counts and classes are done once the kept tokens are known + counted := s.pos + a_count := scratch_take(s, i32, id_count, loc) + b_count := scratch_take(s, i32, id_count, loc) + for &x in a_count { + x = 0 + } + for &x in b_count { + x = 0 + } + for id in a { + a_count[id] += 1 + } + for id in b { + b_count[id] += 1 + } + + // Each side's limit comes from its own whole length + a_limit := max(int) + b_limit := max(int) + if opts.algorithm != .Minimal { + a_limit = min(rough_sqrt(n), PRUNE_MAX_MATCHES) + b_limit = min(rough_sqrt(m), PRUNE_MAX_MATCHES) + } + + a_class := scratch_take(s, Prune_Class, len(a_mid), loc) + b_class := scratch_take(s, Prune_Class, len(b_mid), loc) + classify(a_class, a_mid, b_count, a_limit) + classify(b_class, b_mid, a_count, b_limit) + + a_offset := prefix + b_offset := _inserted_offset(changes) + prefix + kept_n := keep(kept_a, a_mid, a_class, changes.bits, a_offset) + kept_m := keep(kept_b, b_mid, b_class, changes.bits, b_offset) + s.pos = counted + + // Diff what is kept, then map it back + words := scratch_take(s, u64, changes_size(kept_n, kept_m), loc) + for &w in words { + w = 0 + } + kept := changes_from_buffer(kept_n, kept_m, words, loc) + run_algorithm(kept_a[:kept_n], kept_b[:kept_m], id_count, kept, s, opts, loc) + + map_back(changes.bits, a_offset, a_offset + len(a_mid), kept.bits, 0) + map_back(changes.bits, b_offset, b_offset + len(b_mid), kept.bits, _inserted_offset(kept)) + s.pos = start +} diff --git a/core/diff/ratcliff_obershelp.odin b/core/diff/ratcliff_obershelp.odin new file mode 100644 index 000000000..e79685306 --- /dev/null +++ b/core/diff/ratcliff_obershelp.odin @@ -0,0 +1,238 @@ +#+private +package diff + +/* + Ratcliff/Obershelp ("gestalt pattern matching"). Split each region around its longest common block, then + diff both sides the same way. Ties go to the block ending earliest in `a`, then earliest in `b`. + Not minimal, but the matches are long and contiguous. + + Dynamic programming over two rows finds the longest block. `row[j+1]` is the length of the run of matches + ending at the current token of `a` and at `b[j]`. A row holds only the runs reaching it, listed by + position. The rows stay sparse and are cleared through the same lists. + + A token occurring in `b` more than `RO_FREQUENT` times (often `}` or a blank line in code) can carry a run + on but never start one. Its row only continues the runs that reached the token before it. That costs no + more than there are runs, however often the token occurs. Any other token's row starts a run at each of + its occurrences, found through a chain. A block's length counts the frequent tokens inside it. The chosen + block grows back through any equal tokens just before it. A region with only frequent tokens in common + goes to Myers. + + The search can still be quadratic when tokens repeat a lot. The whole diff has a budget of + `RO_STEPS_PER_TOKEN * (n+m)` steps, or `max_cost * (n+m)` when that is set. Myers diffs the regions left + when it runs out, and any region whose search would overrun it. +*/ + +// The default budget for the whole diff, in search steps per token. +RO_STEPS_PER_TOKEN :: 64 + +// A token occurring in `b` more often than this cannot start a block. Histogram's anchors have the same limit. +RO_FREQUENT :: 64 + +// The end of a chain of occurrences. +RO_NONE :: -1 +// +Ratcliff_Obershelp :: struct { + a, b: []i32, + first: []i32, // the first occurrence of each ID in the region of `b` being searched, or `RO_NONE` + next: []i32, // the next occurrence of `b[j]` in that region, or `RO_NONE` + b_count: []i32, // occurrences of each ID in the whole of `b` + prev, cur: []i32, // the rows: the length of the run ending at `b[j]` is at index `j+1` + prev_at: []i32, // where in `b` the runs of `prev` end, the first `prev_len` of them + cur_at: []i32, + prev_len: int, + myers: Myers, + max_cost: int, + work_left: int, +} + +@(require_results) +ratcliff_obershelp_scratch_size :: proc "contextless" (n, m: int, id_count: int) -> int { + chains := scratch_bytes(i32, id_count) + scratch_bytes(i32, m) + counts := scratch_bytes(i32, id_count) + rows := 2*scratch_bytes(i32, m+1) + 2*scratch_bytes(i32, m) + return chains + counts + rows + myers_scratch_size(n, m) +} + +ratcliff_obershelp :: proc(a, b: []i32, id_count: int, changes: Changes, s: ^Scratch, max_cost: int, loc := #caller_location) { + n, m := len(a), len(b) + ro := Ratcliff_Obershelp{ + a = a, + b = b, + first = scratch_take(s, i32, id_count, loc), + next = scratch_take(s, i32, m, loc), + b_count = scratch_take(s, i32, id_count, loc), + prev = scratch_take(s, i32, m+1, loc), + cur = scratch_take(s, i32, m+1, loc), + prev_at = scratch_take(s, i32, m, loc), + cur_at = scratch_take(s, i32, m, loc), + myers = myers_make(a, b, s, loc), + max_cost = max_cost, + work_left = RO_STEPS_PER_TOKEN * (n + m), + } + if max_cost > 0 { + tokens := max(n + m, 1) + ro.work_left = max_cost <= max(int)/tokens ? max_cost*tokens : max(int) + } + for &x in ro.first { + x = RO_NONE + } + for &x in ro.prev { + x = 0 + } + for &x in ro.cur { + x = 0 + } + for &x in ro.b_count { + x = 0 + } + for id in b { + ro.b_count[id] += 1 + } + + stack: [MYERS_MAX_DEPTH]Box + depth := 0 + box := Box{0, n, 0, m, false} + for { + switch { + case box.a_lo == box.a_hi: + _mark_inserted(changes, box.b_lo, box.b_hi) + case box.b_lo == box.b_hi: + _mark_deleted(changes, box.a_lo, box.a_hi) + case: + i, j, k, searched := ratcliff_obershelp_longest_block(&ro, box) + switch { + case !searched: + region_n, region_m := box.a_hi - box.a_lo, box.b_hi - box.b_lo + myers_box(&ro.myers, changes, box, myers_cost(ro.max_cost, region_n, region_m), loc) + + case k == 0: + _mark_deleted(changes, box.a_lo, box.a_hi) + _mark_inserted(changes, box.b_lo, box.b_hi) + + case: + lo := Box{box.a_lo, i, box.b_lo, j, false} + hi := Box{i + k, box.a_hi, j + k, box.b_hi, false} + + // The smaller side first, keeping the stack within `MYERS_MAX_DEPTH` + if box_size(lo) <= box_size(hi) { + stack[depth] = hi + box = lo + } else { + stack[depth] = lo + box = hi + } + depth += 1 + continue + } + } + + if depth == 0 { + break + } + depth -= 1 + box = stack[depth] + } +} + +/* + Returns the longest block `a[i:i+k] == b[j:j+k]` within `box` that starts with a token that is not + frequent. The block is grown back through any equal tokens before it. `k == 0` means there is none. + `searched` is false when the region needs diffing another way: the work budget ran out first, or only + frequent tokens match. +*/ +@(require_results) +ratcliff_obershelp_longest_block :: proc(ro: ^Ratcliff_Obershelp, box: Box) -> (i, j, k: int, searched: bool) { + a, b := ro.a, ro.b + if ro.work_left < 0 { + return + } + + // Chain the occurrences of each token in this region of `b`, in order + for y := box.b_hi - 1; y >= box.b_lo; y -= 1 { + ro.next[y] = ro.first[b[y]] + ro.first[b[y]] = i32(y) + } + + searched = true + frequent_in_common := false + ro.prev_len = 0 + for x in box.a_lo.. RO_FREQUENT: + if ro.first[a[x]] != RO_NONE { + frequent_in_common = true + } + + // Carry on the runs that reached the token before, where `a[x]` matches next + for p in ro.prev_at[:ro.prev_len] { + y := int(p) + 1 + if y < box.b_hi && b[y] == a[x] { + cur_len = ratcliff_obershelp_extend(ro, cur_len, x, y, &i, &j, &k) + } + ro.work_left -= 1 + } + + case: + // Every occurrence starts a run, or carries on the one ending just before it + for y := int(ro.first[a[x]]); y != RO_NONE; y = int(ro.next[y]) { + cur_len = ratcliff_obershelp_extend(ro, cur_len, x, y, &i, &j, &k) + ro.work_left -= 1 + } + } + + // Clear the previous row through its list, then swap the rows + for p in ro.prev_at[:ro.prev_len] { + ro.prev[p + 1] = 0 + } + ro.prev, ro.cur = ro.cur, ro.prev + ro.prev_at, ro.cur_at = ro.cur_at, ro.prev_at + ro.prev_len = cur_len + + if ro.work_left < 0 { + searched = false + break + } + } + + // Leave the rows and chains empty for the next region + for p in ro.prev_at[:ro.prev_len] { + ro.prev[p + 1] = 0 + } + ro.prev_len = 0 + for y in box.b_lo.. 0: + // Take in the equal tokens just before it, frequent ones included + for i > box.a_lo && j > box.b_lo && a[i-1] == b[j-1] { + i -= 1 + j -= 1 + k += 1 + } + + case frequent_in_common: + // Only frequent tokens match. Myers diffs that better than calling it all changes + searched = false + } + } + return +} + +// Records the run through `a[x]` and `b[y]` in the current row, and keeps it if it is the longest so far. +@(require_results) +ratcliff_obershelp_extend :: #force_inline proc "contextless" (ro: ^Ratcliff_Obershelp, cur_len, x, y: int, i, j, k: ^int) -> int { + length := ro.prev[y] + 1 + ro.cur[y + 1] = length + ro.cur_at[cur_len] = i32(y) + if int(length) > k^ { + i^ = x - int(length) + 1 + j^ = y - int(length) + 1 + k^ = int(length) + } + return cur_len + 1 +} diff --git a/core/diff/scratch.odin b/core/diff/scratch.odin new file mode 100644 index 000000000..5ccc4db97 --- /dev/null +++ b/core/diff/scratch.odin @@ -0,0 +1,32 @@ +#+private +package diff + +// Every array carved from scratch starts on this alignment. +SCRATCH_ALIGN :: 8 + +// A bump allocator over the caller's scratch buffer. +Scratch :: struct { + buf: []byte, + pos: int, +} + +// Returns the bytes `scratch_take` uses for `count` values of `T`, rounded so the next array stays aligned. +@(require_results) +scratch_bytes :: #force_inline proc "contextless" ($T: typeid, count: int) -> int { + #assert(align_of(T) <= SCRATCH_ALIGN) + return (count*size_of(T) + SCRATCH_ALIGN-1) &~ (SCRATCH_ALIGN-1) +} + +// Carves `count` values of `T` from the scratch buffer. The values are not zeroed. +@(require_results) +scratch_take :: proc(s: ^Scratch, $T: typeid, count: int, loc := #caller_location) -> []T { + base := uintptr(raw_data(s.buf)) + start := int((base + uintptr(s.pos) + (SCRATCH_ALIGN-1)) &~ (SCRATCH_ALIGN-1) - base) + end := start + scratch_bytes(T, count) + + assert(end <= len(s.buf), "scratch buffer too small", loc) + + s.pos = end + + return ([^]T)(rawptr(base + uintptr(start)))[:count] +} diff --git a/core/diff/text.odin b/core/diff/text.odin new file mode 100644 index 000000000..c723c7f27 --- /dev/null +++ b/core/diff/text.odin @@ -0,0 +1,407 @@ +package diff + +import "base:intrinsics" +import "base:runtime" +import "core:strings" +import "core:unicode" +import "core:unicode/utf8" + +// How text is split into tokens. +Token :: enum u8 { + Line, // up to and including "\n"; the last line may have no "\n" + Word, // a run of letters, digits, and '_', a run of whitespace, or any other single rune + Rune, // one rune; each byte of invalid UTF-8 is a rune of its own +} + +/* + What to ignore when comparing tokens. Each flag applies to every token's text. A final "\n" is never + ignored: a last line without one still differs from the same line with one. +*/ +Text_Flag :: enum u8 { + Ignore_CR, // "a\r\n" equals "a\n" + Ignore_Trailing_Space, // whitespace at the end, before any final "\n" + Ignore_All_Space, // all whitespace, apart from a final "\n" + Ignore_Case, // simple per-rune case folding +} +Text_Flags :: bit_set[Text_Flag; u8] + +// A diff of two strings. It owns only `changes`. `a` and `b` must outlive it. +Text_Diff :: struct { + a, b: string, + token: Token, + changes: Changes, +} + +// A run of whole tokens as one substring. `.Equal` and `.Delete` spans come from `a`. `.Insert` spans come from `b`. +Span :: struct { + op: Op, + text: string, +} + +Span_Iterator :: struct { + d: Text_Diff, + edits: Edit_Iterator, + a_pos, b_pos: int, // byte offset of the next token on each side +} + + +// Returns the number of tokens in `s`: the `n` or `m` used to size `Changes` and scratch. +@(require_results) +token_count :: proc(s: string, token: Token) -> (count: int) { + if token == .Line { + count = strings.count(s, "\n") + if len(s) > 0 && s[len(s) - 1] != '\n' { + count += 1 + } + return + } + if token == .Rune { + // NOTE(bill): Count each byte of an invalid UTF-8 as a rune + return utf8.rune_count_in_string(s) + } + for pos := 0; pos < len(s); pos = _token_end(s, pos, token) { + count += 1 + } + return +} + +/* + Returns the scratch bytes `diff_text_buffer` needs for `n` and `m` tokens. + + The token IDs, and each line's indent with `.Indent_Heuristic`, last the whole diff. The token starts and + the interning table share the rest with the algorithm, one after the other. +*/ +@(require_results) +scratch_size_text :: proc "contextless" (n, m: int, opts := Options{}) -> int { + tokens := n + m + kept := scratch_bytes(i32, tokens) + if .Indent_Heuristic in opts.flags { + kept += scratch_bytes(u8, tokens) + } + interning := scratch_bytes(i32, n + 1) + scratch_bytes(i32, m + 1) + intern_scratch_size(tokens) + return SCRATCH_ALIGN + kept + max(interning, algorithm_scratch_size(n, m, tokens, opts)) +} + +// Diffs `a` and `b` into `changes`, sized for `token_count(a, token)` and `token_count(b, token)`. Never allocates. +@(require_results) +diff_text_buffer :: proc( + a, b: string, + changes: Changes, + scratch: []byte, + token := Token.Line, + flags := Text_Flags{}, + opts := Options{}, + loc := #caller_location, +) -> Text_Diff { + @(require_results) + text_equal_exact :: proc(x, y: string, _: rawptr) -> bool { + return x == y + } + + @(require_results) + text_hash_exact :: proc(x: string, _: rawptr) -> u64 { + x := x + return u64(intrinsics.type_hasher_proc(string)(&x, 0)) + } + + + n, m := changes.n, changes.m + + assert(len(a) < int(max(i32)), "text too long to diff", loc) + assert(len(b) < int(max(i32)), "text too long to diff", loc) + assert(n + m < int(max(i32)), "too many tokens to intern", loc) + + s := Scratch{buf = scratch} + ids := scratch_take(&s, i32, n + m, loc) + + indents: []u8 + if .Indent_Heuristic in opts.flags { + assert(token == .Line, ".Indent_Heuristic needs a line diff", loc) + indents = scratch_take(&s, u8, n + m, loc) + } + + interning_start := s.pos + a_tokens := Text_Tokens{a, scratch_take(&s, i32, n + 1, loc)} + b_tokens := Text_Tokens{b, scratch_take(&s, i32, m + 1, loc)} + + _token_starts(a, token, a_tokens.starts, loc) + _token_starts(b, token, b_tokens.starts, loc) + if indents != nil { + for k in 0.. (d: Text_Diff, err: runtime.Allocator_Error) { + n := token_count(a, token) + m := token_count(b, token) + c, scratch := _changes_and_scratch_make( + n, + m, + scratch_size_text(n, m, opts), + allocator, + transient_allocator, + loc, + ) or_return + d = diff_text_buffer(a, b, c, scratch, token, flags, opts, loc) + delete(scratch, transient_allocator, loc) + return +} + +// Frees `d` with the allocator it was made with. The texts it refers to are not its own. +text_diff_destroy :: proc(d: Text_Diff, allocator: runtime.Allocator, loc := #caller_location) { + changes_destroy(d.changes, allocator, loc) +} + + +@(require_results) +span_iterator_make :: proc "contextless" (d: Text_Diff) -> Span_Iterator { + return {d = d, edits = edit_iterator_make(d.changes)} +} + +/* + Iterates over spans in order. Never allocates. + + Usage: + it := diff.span_iterator_make(d) + for s in diff.span_iterate(&it) { + ... + } +*/ +@(require_results) +span_iterate :: proc(it: ^Span_Iterator) -> (s: Span, ok: bool) { + e := edit_iterate(&it.edits) or_return + d := it.d + + a_start := it.a_pos + b_start := it.b_pos + it.a_pos = _skip_tokens(d.a, it.a_pos, e.a_hi - e.a_lo, d.token) + it.b_pos = _skip_tokens(d.b, it.b_pos, e.b_hi - e.b_lo, d.token) + + switch e.op { + case .Equal, .Delete: + s = {e.op, d.a[a_start:it.a_pos]} + case .Insert: + s = {e.op, d.b[b_start:it.b_pos]} + } + return s, true +} + + +// Returns where the token starting at `s[pos]` ends. +@(require_results) +_token_end :: proc(s: string, pos: int, token: Token) -> int { + Word_Class :: enum u8 { + Other, + Word, + Space, + } + + @(require_results) + word_class :: #force_inline proc(r: rune) -> Word_Class { + switch r { + case '_', 'a'..='z', 'A'..='Z', '0'..='9': + return .Word + case '\t', '\n', '\v', '\f', '\r', ' ': + return .Space + } + switch { + case r < utf8.RUNE_SELF: + return .Other + case unicode.is_letter(r) || unicode.is_digit(r): + return .Word + case unicode.is_space(r): + return .Space + } + return .Other + } + + + switch token { + case .Line: + if i := strings.index_byte(s[pos:], '\n'); i >= 0 { + return pos + i + 1 + } + return len(s) + + case .Rune: + if s[pos] < utf8.RUNE_SELF { + return pos + 1 + } + _, width := utf8.decode_rune_in_string(s[pos:]) + return pos + width + + case .Word: + // NOTE(bill): ASCII without decoding because most text will be mostly ASCII in practice with what gets diffed + r, width := rune(s[pos]), 1 + if r >= utf8.RUNE_SELF { + r, width = utf8.decode_rune_in_string(s[pos:]) + } + class := word_class(r) + end := pos + width + if class == .Other { + return end + } + for end < len(s) { + next, next_width := rune(s[end]), 1 + if next >= utf8.RUNE_SELF { + next, next_width = utf8.decode_rune_in_string(s[end:]) + } + if word_class(next) != class { + break + } + end += next_width + } + return end + } + return len(s) +} + +// Returns where the `count` tokens from `s[pos]` end. +@(require_results) +_skip_tokens :: proc(s: string, pos, count: int, token: Token) -> int { + pos := pos + for _ in 0.. bool { + flags := (^Text_Flags)(user_data)^ + rx := text_reader_make(x, flags) + ry := text_reader_make(y, flags) + for { + cx, ok_x := text_reader_next(&rx) + cy, ok_y := text_reader_next(&ry) + if ok_x != ok_y || cx != cy { + return false + } + if !ok_x { + return true + } + } +} + +// FNV-1a over the runes `Text_Reader` gives, with `user_data` pointing at the `Text_Flags`. +@(require_results) +_text_hash :: proc(x: string, user_data: rawptr) -> u64 { + flags := (^Text_Flags)(user_data)^ + r := text_reader_make(x, flags) + h := u64(0xcbf29ce484222325) + for c in text_reader_next(&r) { + h = (h ~ u64(c)) * 0x100000001b3 + } + return h +} + +// Reads a token's runes with `Text_Flags` applied: whitespace and carriage returns dropped, case folded. +@(private) +Text_Reader :: struct { + s: string, + pos: int, + end: int, // where the runes to read stop, before any final "\n" and whatever is ignored before it + newline: bool, // a final "\n" is still to be read + flags: Text_Flags, +} + +@(private, require_results) +text_reader_make :: proc(s: string, flags: Text_Flags) -> (r: Text_Reader) { + r.s, r.flags, r.end = s, flags, len(s) + if r.end > 0 && s[r.end - 1] == '\n' { + r.newline = true + r.end -= 1 + } + if .Ignore_CR in flags && r.newline && r.end > 0 && s[r.end - 1] == '\r' { + r.end -= 1 + } + if flags & {.Ignore_Trailing_Space, .Ignore_All_Space} != nil { + for r.end > 0 { + c, width := utf8.decode_last_rune_in_string(s[:r.end]) + if !unicode.is_space(c) { + break + } + r.end -= width + } + } + return +} + +@(private, require_results) +text_reader_next :: proc(r: ^Text_Reader) -> (c: rune, ok: bool) { + // The smallest rune that `c` equals under simple case folding. Runes that fold together fold to the same one + @(require_results) + fold_rune :: proc(c: rune) -> rune { + if c < utf8.RUNE_SELF { + if 'a' <= c && c <= 'z' { + return c - ('a' - 'A') + } + return c + } + smallest := c + for f := unicode.simple_fold(c); f != c; f = unicode.simple_fold(f) { + smallest = min(smallest, f) + } + return smallest + } + + + for r.pos < r.end { + width: int + c, width = utf8.decode_rune_in_string(r.s[r.pos:r.end]) + r.pos += width + if .Ignore_All_Space in r.flags && unicode.is_space(c) { + continue + } + if .Ignore_Case in r.flags { + c = fold_rune(c) + } + return c, true + } + if r.newline { + r.newline = false + return '\n', true + } + return +} diff --git a/core/diff/unified.odin b/core/diff/unified.odin new file mode 100644 index 000000000..3241165cb --- /dev/null +++ b/core/diff/unified.odin @@ -0,0 +1,372 @@ +package diff + +import "base:runtime" +import "core:io" +import "core:strings" +import "core:terminal/ansi" + +Unified_Kind :: enum u8 { + Hunk_Header, // "@@ -a,b +c,d @@"; the ranges are in `Unified_Line.hunk` + Equal, // " line" + Delete, // "-line" + Insert, // "+line" + No_Newline, // "\ No newline at end of file", after the line it applies to +} + +Unified_Line :: struct { + kind: Unified_Kind, + text: string, // the line without its prefix or newline; empty for `.Hunk_Header` and `.No_Newline` + hunk: Hunk, // only set for `.Hunk_Header` + a_line, b_line: int, // 1-based line numbers; 0 when the line is not on that side +} + +Unified_Iterator :: struct { + d: Text_Diff, + hunks: Hunk_Iterator, + edits: Edit_Iterator, + edit: Edit, // the edit whose lines are being given + a_index, b_index: int, // the next line on each side + a_pos, b_pos: int, // byte offset of that line + in_hunk: bool, + no_newline: bool, // a `.No_Newline` line is due +} + +// Escape sequences written before each kind of output, and `reset` after it. All empty means no colour. +Style :: struct { + file_header: string, + hunk_header: string, + equal: string, + delete: string, + insert: string, + delete_emphasis: string, // the changed part within a deleted line + insert_emphasis: string, // the changed part within an inserted line + reset: string, +} + +STYLE_NONE :: Style{} + +// Colours similar to git's defaults. +STYLE_ANSI :: Style{ + file_header = ansi.CSI + ansi.BOLD + ansi.SGR, + hunk_header = ansi.CSI + ansi.FG_CYAN + ansi.SGR, + delete = ansi.CSI + ansi.FG_RED + ansi.SGR, + insert = ansi.CSI + ansi.FG_GREEN + ansi.SGR, + delete_emphasis = ansi.CSI + ansi.FG_RED + ";" + ansi.INVERT + ansi.SGR, + insert_emphasis = ansi.CSI + ansi.FG_GREEN + ";" + ansi.INVERT + ansi.SGR, + reset = ansi.CSI + ansi.RESET + ansi.SGR, +} + +// How the changed part within a changed line is highlighted. +Emphasis :: enum u8 { + None, + Prefix_Suffix, // trim the common prefix and suffix of paired deleted and inserted lines; needs no scratch + Words, // word diff of each block of deleted and inserted lines; needs `emphasis_scratch_size` bytes of scratch +} + +/* + How `unified` and `write_unified_to_builder` make room for the output. + + - `.Exact` writes the diff twice: once to count its bytes, then into one allocation of exactly that size. + Nothing is resized. Suits any allocator, including fixed buffers and bump arenas. + - `.Grow` writes the diff once into growing memory, then shrinks it to fit. Close to twice as fast. + Suits the heap and allocators that resize their last allocation in place: `core:mem`'s `Arena`, `Stack`, + and `Small_Stack`, and `core:mem/virtual`'s `Arena` while its current block has room. With nothing else + allocated from them meanwhile, these end up holding exactly the output. Inside a temporary region of + either `Arena`, the shrink cannot give memory back: up to as much again stays in use until the region + ends. A `Dynamic_Arena` copies an allocation larger than its out-of-band size to grow it, and keeps every + copy until it is reset. Use `.Exact` with it. +*/ +Sizing :: enum u8 { + Exact, + Grow, +} + + +// Makes an iterator over the lines of unified output, without the "---" and "+++" file headers. `d` must be a line diff. +@(require_results) +unified_iterator_make :: proc(d: Text_Diff, context_len := DEFAULT_CONTEXT_LEN, loc := #caller_location) -> Unified_Iterator { + assert(d.token == .Line, "unified output needs a line diff", loc) + return {d = d, hunks = hunk_iterator_make(d.changes, context_len)} +} + +/* + Iterates over the lines of unified output in order. Never allocates. + + Usage: + it := diff.unified_iterator_make(d) + for line in diff.unified_iterate(&it) { + switch line.kind { + case .Hunk_Header: ... + case .Equal: ... + case .Delete: ... + case .Insert: ... + case .No_Newline: ... + } + } +*/ +@(require_results) +unified_iterate :: proc(it: ^Unified_Iterator) -> (line: Unified_Line, ok: bool) { + @(require_results) + next_line :: proc(s: string, pos, index: ^int) -> (text: string, no_newline: bool) { + start := pos^ + pos^ = _token_end(s, start, .Line) + index^ += 1 + text = s[start:pos^] + if len(text) > 0 && text[len(text) - 1] == '\n' { + return text[:len(text) - 1], false + } + return text, true + } + + + if it.no_newline { + it.no_newline = false + return {kind = .No_Newline}, true + } + + d := it.d + for { + if !it.in_hunk { + h := hunk_iterate(&it.hunks) or_return + it.a_pos = _skip_tokens(d.a, it.a_pos, h.a_lo - it.a_index, .Line) + it.b_pos = _skip_tokens(d.b, it.b_pos, h.b_lo - it.b_index, .Line) + it.a_index = h.a_lo + it.b_index = h.b_lo + it.edits = edit_iterator_make_hunk(d.changes, h) + it.edit = {} + it.in_hunk = true + return {kind = .Hunk_Header, hunk = h}, true + } + + e := it.edit + switch e.op { + case .Equal: + if it.a_index < e.a_hi { + line = { + kind = .Equal, + a_line = it.a_index + 1, + b_line = it.b_index + 1, + } + line.text, it.no_newline = next_line(d.a, &it.a_pos, &it.a_index) + _, _ = next_line(d.b, &it.b_pos, &it.b_index) + return line, true + } + case .Delete: + if it.a_index < e.a_hi { + line = { + kind = .Delete, + a_line = it.a_index + 1, + } + line.text, it.no_newline = next_line(d.a, &it.a_pos, &it.a_index) + return line, true + } + case .Insert: + if it.b_index < e.b_hi { + line = { + kind = .Insert, + b_line = it.b_index + 1, + } + line.text, it.no_newline = next_line(d.b, &it.b_pos, &it.b_index) + return line, true + } + } + + // This edit is used up: move on to the next, or out of the hunk + next, has_next := edit_iterate(&it.edits) + if !has_next { + it.in_hunk = false + continue + } + it.edit = next + } +} + +// Writes `d` as a unified diff, file headers included. Writes nothing without changes. `d` must be a line diff. Never allocates. +write_unified :: proc( + w: io.Writer, + d: Text_Diff, + a_name := "a", + b_name := "b", + context_len := DEFAULT_CONTEXT_LEN, + style := STYLE_NONE, + emphasis := Emphasis.None, + scratch: []byte = nil, + loc := #caller_location, +) -> io.Error { + write_styled :: proc(w: io.Writer, color, reset, prefix, text: string) -> io.Error { + if color != "" { + io.write_string(w, color) or_return + } + io.write_string(w, prefix) or_return + io.write_string(w, text) or_return + if color != "" { + io.write_string(w, reset) or_return + } + return io.write_byte(w, '\n') + } + + // GNU diff style ranges + write_range :: proc(w: io.Writer, lo, hi: int) -> io.Error { + count := hi - lo + io.write_int(w, count == 0 ? lo : lo + 1) or_return + if count != 1 { + io.write_byte(w, ',') or_return + io.write_int(w, count) or_return + } + return nil + } + + // Without an emphasis colour there is nothing to show + emphasis := emphasis + if style.delete_emphasis == "" && style.insert_emphasis == "" { + emphasis = .None + } + + it := unified_iterator_make(d, context_len, loc) + block: Emphasis_Block + headers_written := false + for line in unified_iterate(&it) { + if !headers_written { + write_styled(w, style.file_header, style.reset, "--- ", a_name) or_return + write_styled(w, style.file_header, style.reset, "+++ ", b_name) or_return + headers_written = true + } + + switch line.kind { + case .Hunk_Header: + block = {} + if style.hunk_header != "" { + io.write_string(w, style.hunk_header) or_return + } + io.write_string(w, "@@ -") or_return + write_range(w, line.hunk.a_lo, line.hunk.a_hi) or_return + io.write_string(w, " +") or_return + write_range(w, line.hunk.b_lo, line.hunk.b_hi) or_return + io.write_string(w, " @@") or_return + if style.hunk_header != "" { + io.write_string(w, style.reset) or_return + } + io.write_byte(w, '\n') or_return + case .Equal: + block = {} + write_styled(w, style.equal, style.reset, " ", line.text) or_return + case .Delete: + if emphasis != .None && line.a_line - 1 == it.edit.a_lo { + block = emphasis_block_make(&it, line, emphasis, scratch, loc) + } + if block.active { + write_emphasized(w, &block, .Delete, line.text, style) or_return + } else { + write_styled(w, style.delete, style.reset, "-", line.text) or_return + } + case .Insert: + if block.active { + write_emphasized(w, &block, .Insert, line.text, style) or_return + } else { + write_styled(w, style.insert, style.reset, "+", line.text) or_return + } + case .No_Newline: + io.write_string(w, "\\ No newline at end of file\n") or_return + } + } + return nil +} + +/* + Appends `d` to `sb` as a unified diff, file headers included. `d` must be a line diff. + + `.Exact` grows `sb` once, by exactly `unified_size` bytes. `.Grow` grows it while writing. + If `sb` cannot grow enough, nothing is appended. +*/ +write_unified_to_builder :: proc( + sb: ^strings.Builder, + d: Text_Diff, + a_name := "a", + b_name := "b", + context_len := DEFAULT_CONTEXT_LEN, + style := STYLE_NONE, + emphasis := Emphasis.None, + scratch: []byte = nil, + sizing := Sizing.Exact, + loc := #caller_location, +) -> runtime.Allocator_Error { + switch sizing { + case .Exact: + size := unified_size(d, a_name, b_name, context_len, style, emphasis, scratch, loc) + reserve(&sb.buf, len(sb.buf) + size, loc) or_return + err := write_unified(strings.to_writer(sb), d, a_name, b_name, context_len, style, emphasis, scratch, loc) + assert(err == nil, "unified output outgrew its own size", loc) + + case .Grow: + // A builder only fails to take a write when it cannot grow + start := len(sb.buf) + if write_unified(strings.to_writer(sb), d, a_name, b_name, context_len, style, emphasis, scratch, loc) != nil { + resize(&sb.buf, start, loc) + return .Out_Of_Memory + } + } + return nil +} + +// Returns the exact number of bytes `write_unified` would write. +@(require_results) +unified_size :: proc( + d: Text_Diff, + a_name := "a", + b_name := "b", + context_len := DEFAULT_CONTEXT_LEN, + style := STYLE_NONE, + emphasis := Emphasis.None, + scratch: []byte = nil, + loc := #caller_location, +) -> int { + // With no stream to pass them on to, the writes are only counted + counter: io.Counting_Stream + _ = write_unified(io.counting_stream_init(&counter, {}), d, a_name, b_name, context_len, style, emphasis, scratch, loc) + return int(counter.bytes_written) +} + +/* + Returns `d` as a unified diff, allocated from `allocator`. Free it with the same allocator. + + `.Exact` makes one allocation of exactly `unified_size` bytes. `.Grow` grows the memory while writing, + then shrinks it to fit. On failure, nothing stays allocated. +*/ +@(require_results) +unified :: proc( + d: Text_Diff, + allocator: runtime.Allocator, + a_name := "a", + b_name := "b", + context_len := DEFAULT_CONTEXT_LEN, + style := STYLE_NONE, + emphasis := Emphasis.None, + scratch: []byte = nil, + sizing := Sizing.Exact, + loc := #caller_location, +) -> (s: string, err: runtime.Allocator_Error) { + switch sizing { + case .Exact: + size := unified_size(d, a_name, b_name, context_len, style, emphasis, scratch, loc) + buf := make([]byte, size, allocator, loc) or_return + sb := strings.builder_from_bytes(buf) + write_err := write_unified(strings.to_writer(&sb), d, a_name, b_name, context_len, style, emphasis, scratch, loc) + assert(write_err == nil, "unified output outgrew its own size", loc) + assert(len(sb.buf) == size, "unified output outgrew its own size", loc) + return string(buf), nil + + case .Grow: + sb := strings.builder_make(allocator, loc) or_return + err = write_unified_to_builder(&sb, d, a_name, b_name, context_len, style, emphasis, scratch, .Grow, loc) + if err == nil { + // Fitted, the memory is exactly what is freed with the string + _, err = shrink(&sb.buf, loc = loc) + } + if err != nil { + strings.builder_destroy(&sb) + return "", err + } + return strings.to_string(sb), nil + } + return +} diff --git a/core/io/util.odin b/core/io/util.odin index 35297c121..10f0ae11a 100644 --- a/core/io/util.odin +++ b/core/io/util.odin @@ -516,3 +516,100 @@ _section_reader_proc :: proc(stream_data: rawptr, mode: Stream_Mode, p: []byte, return 0, nil } + + +// Counting_Stream passes everything on to an underlying stream, counting the bytes read and written and how +// often each mode is used. +// +// With a nil underlying stream it takes every write and discards it, which measures output without keeping it. +// It then seeks as a file would, keeping `pos` and `size`, so `size` is how large a file the writes would have made +// even when they seek back or write at an offset; reads always give .EOF, as nothing written is kept; +// and flushing, closing and destroying do nothing. +Counting_Stream :: struct { + s: Stream, // underlying stream, or nil to discard writes + bytes_read: i64, // by .Read and .Read_At, as the underlying stream reports them + bytes_written: i64, // by .Write and .Write_At, as the underlying stream reports them + calls: [Stream_Mode]i64, // how many times each mode was used + + // Only with a nil underlying stream + pos: i64, // where the next .Write lands + size: i64, // the end of the furthest write +} + +@(private) +_counting_stream_proc :: proc(stream_data: rawptr, mode: Stream_Mode, p: []byte, offset: i64, whence: Seek_From) -> (n: i64, err: Error) { + c := (^Counting_Stream)(stream_data) + c.calls[mode] += 1 + + if c.s.procedure == nil { + switch mode { + case .Write: + c.bytes_written += i64(len(p)) + c.pos += i64(len(p)) + c.size = max(c.size, c.pos) + return i64(len(p)), nil + + case .Write_At: + if offset < 0 { + return 0, .Invalid_Offset + } + c.bytes_written += i64(len(p)) + c.size = max(c.size, offset + i64(len(p))) + return i64(len(p)), nil + + case .Read, .Read_At: + return 0, .EOF + + case .Seek: + pos: i64 + switch whence { + case .Start: pos = offset + case .Current: pos = c.pos + offset + case .End: pos = c.size + offset + case: return 0, .Invalid_Whence + } + if pos < 0 { + return 0, .Invalid_Offset + } + c.pos = pos + return pos, nil + + case .Size: + return c.size, nil + + case .Flush, .Close, .Destroy: + return 0, nil + + case .Query: + return query_utility({.Write, .Write_At, .Read, .Read_At, .Seek, .Size, .Flush, .Close, .Destroy, .Query}) + } + return 0, .Unsupported + } + + if mode == .Query { + return query_utility(query(c.s) + {.Query}) + } + + // Bytes are counted as the underlying stream reports them, errors included, so a short write counts what went through + n, err = c.s.procedure(c.s.data, mode, p, offset, whence) + #partial switch mode { + case .Read, .Read_At: + c.bytes_read += n + case .Write, .Write_At: + c.bytes_written += n + } + return +} + +// counting_stream_init returns a Stream that counts what passes through it to 's', with its counts reset. +// A nil 's' discards every write, keeping only where it would have landed. +counting_stream_init :: proc(c: ^Counting_Stream, s: Stream) -> Stream { + c^ = {s = s} + return counting_stream_to_stream(c) +} + +counting_stream_to_stream :: proc(c: ^Counting_Stream) -> (s: Stream) { + s.data = c + s.procedure = _counting_stream_proc + return +} diff --git a/examples/all/all_main.odin b/examples/all/all_main.odin index c14dcf9ad..5a8f9f140 100644 --- a/examples/all/all_main.odin +++ b/examples/all/all_main.odin @@ -67,6 +67,8 @@ package all @(require) import "core:debug/pe" @(require) import "core:debug/trace" +@(require) import "core:diff" + @(require) import "core:dynlib" @(require) import "core:net" diff --git a/tests/benchmark/all.odin b/tests/benchmark/all.odin index 042b2fa63..92d36213c 100644 --- a/tests/benchmark/all.odin +++ b/tests/benchmark/all.odin @@ -2,6 +2,7 @@ package benchmarks @(require) import "bytes" @(require) import "crypto" +@(require) import "diff" @(require) import "hash" @(require) import "math" @(require) import "text/regex" diff --git a/tests/benchmark/diff/benchmark_diff.odin b/tests/benchmark/diff/benchmark_diff.odin new file mode 100644 index 000000000..368851fad --- /dev/null +++ b/tests/benchmark/diff/benchmark_diff.odin @@ -0,0 +1,317 @@ +package benchmark_core_diff + +import "core:diff" +import "core:fmt" +import "core:log" +import "core:math/rand" +import "core:strings" +import "core:testing" +import "core:text/table" +import "core:time" + +// Each diff is timed this many times, and the fastest run is reported. +RUNS :: 3 + +@(rodata) +VARIANTS := [?]diff.Options{ + {algorithm = .Myers}, + {algorithm = .Myers, flags = {.Prune}}, + {algorithm = .Minimal}, + {algorithm = .Histogram}, + {algorithm = .Histogram, flags = {.Prune}}, + {algorithm = .Ratcliff_Obershelp}, + {algorithm = .Ratcliff_Obershelp, flags = {.Prune}}, + {algorithm = .Myers, flags = {.Prune, .Compact}}, // as git --no-indent-heuristic + {algorithm = .Histogram, flags = {.Compact}}, // as git --histogram --no-indent-heuristic + {algorithm = .Myers, flags = {.Prune, .Compact, .Indent_Heuristic}}, // as plain git diff, for lines only +} + +// Short and common, meaning lines made of them repeat a lot. +@(rodata) +CODE_WORDS := [?]string{"if", "for", "x", "y", "return", "count", "}", "{", "\t", "// note", "foo(bar)", "Value"} + +// Enough words for prose to read as text, with plenty repeated. +@(rodata) +PROSE_WORDS := [?]string{ + "the", "a", "of", "and", "to", "in", "is", "that", "it", "was", "for", "on", "are", "with", "as", "they", + "be", "at", "one", "have", "this", "from", "by", "hot", "word", "but", "what", "some", "we", "can", "out", + "other", "were", "all", "there", "when", "up", "use", "your", "how", "said", "an", "each", "she", "which", + "do", "their", "time", "if", "will", "way", "about", "many", "then", "them", "write", "would", "like", "so", +} + +Scenario :: struct { + name: string, + a, b: string, + token: diff.Token, +} + +// Lines that are nearly all different from each other, like most source code. +unique_line :: proc(sb: ^strings.Builder) { + fmt.sbprintf(sb, "\tvalue_%d := compute(%d, %d)\n", rand.int_max(1_000_000), rand.int_max(1000), rand.int_max(1000)) +} + +// Lines made of a few common words, which repeat many times over. +repetitive_line :: proc(sb: ^strings.Builder) { + for _ in 0..<1 + rand.int_max(4) { + strings.write_string(sb, CODE_WORDS[rand.int_max(len(CODE_WORDS))]) + strings.write_byte(sb, ' ') + } + strings.write_byte(sb, '\n') +} + +// Two texts of `lines` lines, where about one line in `every` is replaced, inserted or removed. +edited_lines :: proc(lines, every: int, make_line: proc(sb: ^strings.Builder)) -> (a, b: string) { + sa := strings.builder_make(context.temp_allocator) + sb := strings.builder_make(context.temp_allocator) + for _ in 0.. (a, b: string) { + sa := strings.builder_make(context.temp_allocator) + prose(&sa, words) + a = strings.to_string(sa) + + sb := strings.builder_make(context.temp_allocator) + for word in strings.split_after(a, " ", context.temp_allocator) { + if rand.int_max(every) == 0 { + strings.write_string(&sb, PROSE_WORDS[rand.int_max(len(PROSE_WORDS))]) + strings.write_byte(&sb, ' ') + } else { + strings.write_string(&sb, word) + } + } + return a, strings.to_string(sb) +} + +lines_scenario :: proc(name: string, lines, every: int, make_line: proc(sb: ^strings.Builder)) -> (sc: Scenario) { + sc.name, sc.token = name, .Line + sc.a, sc.b = edited_lines(lines, every, make_line) + return +} + +// Two texts of unique lines made separately, meaning they have nothing in common. +unrelated_scenario :: proc(name: string, lines: int) -> (sc: Scenario) { + sc.name, sc.token = name, .Line + sc.a, _ = edited_lines(lines, 1, unique_line) + sc.b, _ = edited_lines(lines, 1, unique_line) + return +} + +prose_scenario :: proc(name: string, words, every: int, token: diff.Token) -> (sc: Scenario) { + sc.name, sc.token = name, token + sc.a, sc.b = edited_prose(words, every) + return +} + +// Times `opts` on `sc`, returning the fastest of `RUNS`, how many tokens changed, and the scratch it needed. +time_diff :: proc(sc: Scenario, opts: diff.Options) -> (fastest: time.Duration, changed, scratch: int) { + for run in 0.. string { + sb := strings.builder_make(context.temp_allocator) + for flag in flags { + if len(sb.buf) > 0 { + strings.write_string(&sb, ", ") + } + fmt.sbprint(&sb, flag) + } + return strings.to_string(sb) +} + +report :: proc(title: string, tbl: ^table.Table) { + sb := strings.builder_make() + defer strings.builder_destroy(&sb) + fmt.sbprintln(&sb, title) + table.write_plain_table(strings.to_writer(&sb), tbl) + log.info(strings.to_string(sb)) +} + +@test +benchmark_algorithms :: proc(t: ^testing.T) { + rand.reset(t.seed) + + scenarios := [?]Scenario{ + lines_scenario("lines, unique, 0.5% edited", 100_000, 200, unique_line), + lines_scenario("lines, repetitive, 0.5% edited", 100_000, 200, repetitive_line), + lines_scenario("lines, unique, 10% edited", 20_000, 10, unique_line), + unrelated_scenario("lines, nothing in common", 5_000), + prose_scenario("words, prose, 2% edited", 50_000, 50, .Word), + prose_scenario("runes, prose, 2% edited", 5_000, 50, .Rune), + } + + tbl: table.Table + table.init(&tbl) + defer table.destroy(&tbl) + table.aligned_header_of_values(&tbl, .Right, "Input", "Tokens", "Algorithm", "Flags", "Fastest", "Changed", "Scratch") + + for sc in scenarios { + tokens := diff.token_count(sc.a, sc.token) + diff.token_count(sc.b, sc.token) + for opts in VARIANTS { + if .Indent_Heuristic in opts.flags && sc.token != .Line { + continue // only lines have indentation + } + fastest, changed, scratch := time_diff(sc, opts) + table.aligned_row_of_values(&tbl, .Right, + sc.name, + tokens, + opts.algorithm, + flags_name(opts.flags), + fmt.tprintf("%v", fastest), + changed, + fmt.tprintf("%.1f MB", f64(scratch) / 1e6), + ) + } + } + report("core:diff algorithms, the fastest of 3 runs each", &tbl) +} + +@test +benchmark_generic :: proc(t: ^testing.T) { + rand.reset(t.seed) + + n := 1_000_000 + a := make([]int, n) + defer delete(a) + for &x in a { + x = rand.int_max(100_000) + } + b := make([dynamic]int, 0, n + 1_000) + defer delete(b) + append(&b, ..a) + for _ in 0..<1_000 { + switch rand.int_max(3) { + case 0: inject_at(&b, rand.int_max(len(b) + 1), rand.int_max(100_000)) + case 1: ordered_remove(&b, rand.int_max(len(b))) + case 2: b[rand.int_max(len(b))] = rand.int_max(100_000) + } + } + + tbl: table.Table + table.init(&tbl) + defer table.destroy(&tbl) + table.aligned_header_of_values(&tbl, .Right, "Algorithm", "Flags", "Fastest", "Changed", "Scratch") + + for opts in VARIANTS { + switch { + case opts.algorithm == .Ratcliff_Obershelp: + continue // random integers repeat too often for it to be worth timing + case .Indent_Heuristic in opts.flags: + continue // only text has indentation + } + + fastest: time.Duration + changed: int + for run in 0.. T { + assert(err == nil, "allocation failed", loc) + return value +} + +// Length of the longest common subsequence, by dynamic programming over two rows of `len(b)+1` ints each. +lcs_length :: proc(a, b: []i32, rows: []int = nil) -> int { + if rows == nil { + owned := make([]int, 2*(len(b) + 1)) + defer delete(owned) + return lcs_length(a, b, owned) + } + + prev := rows[:len(b) + 1] + curr := rows[len(b) + 1:][:len(b) + 1] + for &x in prev { + x = 0 + } + curr[0] = 0 + + for x in a { + for y, j in b { + if x == y { + curr[j+1] = prev[j] + 1 + } else { + curr[j+1] = max(prev[j+1], curr[j]) + } + } + prev, curr = curr, prev + } + return prev[len(b)] +} + +// The cost of a minimal diff of `a` and `b`: tokens deleted plus tokens inserted. +minimal_cost :: proc(a, b: []i32, rows: []int = nil) -> int { + return len(a) + len(b) - 2*lcs_length(a, b, rows) +} + +cost :: proc(c: diff.Changes) -> int { + s := diff.stats(c) + return s.deleted + s.inserted +} + +/* + Checks that `c` is a valid diff of `a` and `b`: the tokens left unmarked on each side are the same sequence, + so `b` can be rebuilt from `a`, and no bit past the end of either side is set. +*/ +check_valid :: proc(t: ^testing.T, a, b: []i32, c: diff.Changes, loc := #caller_location) -> bool { + if !testing.expect(t, c.n == len(a) && c.m == len(b), "changes sized for different inputs", loc = loc) { + return false + } + if !testing.expect(t, len(c.bits) == diff.changes_size(c.n, c.m), "changes have the wrong number of words", loc = loc) { + return false + } + + i, j := 0, 0 + for { + for i < len(a) && diff.is_deleted(c, i) { + i += 1 + } + for j < len(b) && diff.is_inserted(c, j) { + j += 1 + } + if i == len(a) || j == len(b) { + break + } + if a[i] != b[j] { + testing.expectf(t, false, "kept a[%d] = %d does not match kept b[%d] = %d (a = %v, b = %v)", i, a[i], j, b[j], a, b, loc = loc) + return false + } + i += 1 + j += 1 + } + if i != len(a) || j != len(b) { + testing.expectf(t, false, "kept tokens left over: a[%d:], b[%d:] (a = %v, b = %v)", i, j, a, b, loc = loc) + return false + } + + a_words := (c.n + 63) / 64 + b_words := (c.m + 63) / 64 + for k in c.n..> 6] & (u64(1) << uint(k & 63)) != 0 { + testing.expectf(t, false, "padding bit %d set on the a side", k, loc = loc) + return false + } + } + for k in c.m..> 6] & (u64(1) << uint(bit & 63)) != 0 { + testing.expectf(t, false, "padding bit %d set on the b side", k, loc = loc) + return false + } + } + return true +} + +@(require_results) +id_count_of :: proc(a, b: []i32) -> (count: int) { + for x in a { + count = max(count, int(x) + 1) + } + for x in b { + count = max(count, int(x) + 1) + } + return +} + +// Diffs with scratch of exactly the size asked for. +@(require_results) +diff_exact :: proc(a, b: []i32, opts := diff.Options{}) -> diff.Changes { + id_count := id_count_of(a, b) + c := must(diff.changes_make(len(a), len(b), context.allocator)) + + scratch := make([]byte, diff.scratch_size_interned(len(a), len(b), id_count, opts)) + defer delete(scratch) + + diff.diff_interned_buffer(a, b, id_count, c, scratch, opts) + return c +} + +// The options every pair is checked under: exact, default cap, caps small enough to be hit at once, and the other algorithms. +@(rodata) +PAIR_OPTIONS := [?]diff.Options{ + {algorithm = .Minimal}, + {algorithm = .Minimal, max_cost = 1}, // `.Minimal` ignores the cap + {algorithm = .Myers}, + {algorithm = .Myers, max_cost = 1}, + {algorithm = .Myers, max_cost = 3}, + {algorithm = .Histogram}, + {algorithm = .Histogram, max_cost = 1}, // for its fallback to Myers + {algorithm = .Ratcliff_Obershelp}, + {algorithm = .Ratcliff_Obershelp, max_cost = 1}, // a budget so small that Myers takes over + {algorithm = .Myers, flags = {.Prune}}, + {algorithm = .Minimal, flags = {.Prune}}, + {algorithm = .Histogram, flags = {.Prune}}, + {algorithm = .Ratcliff_Obershelp, flags = {.Prune}}, + {algorithm = .Myers, flags = {.Compact}}, // sliding changes never changes how many there are + {algorithm = .Minimal, flags = {.Compact}}, + {algorithm = .Histogram, flags = {.Compact}}, + {algorithm = .Myers, flags = {.Prune, .Compact}}, + + // Combinations git has no equivalent for + {algorithm = .Histogram, flags = {.Compact}, max_cost = 1}, // merged groups are diffed again under the cap + {algorithm = .Histogram, flags = {.Prune, .Compact}}, + {algorithm = .Ratcliff_Obershelp, flags = {.Compact}}, + {algorithm = .Ratcliff_Obershelp, flags = {.Prune, .Compact}}, +} + +/* + Whether the diff must be minimal. `.Myers` with its default cap is minimal as long as the search never reaches + 256 edits, where the cap and the snake heuristic start: true for any minimal diff of up to 2*256 tokens. + `.Prune` keeps `.Minimal` minimal, but can set aside tokens `.Myers` would have matched. + The other algorithms are never promised to be minimal. +*/ +must_be_minimal :: proc(opts: diff.Options, expected: int) -> bool { + switch opts.algorithm { + case .Minimal: + return true + + case .Myers: + return opts.max_cost == 0 && + .Prune not_in opts.flags && + expected <= 2*256 + + case .Histogram, .Ratcliff_Obershelp: + return false + } + return false +} + +// Checks the cost of a valid diff: minimal where it must be, never below minimal. +check_cost :: proc(t: ^testing.T, a, b: []i32, c: diff.Changes, opts: diff.Options, expected: int, loc := #caller_location) -> bool { + if must_be_minimal(opts, expected) { + return testing.expectf(t, cost(c) == expected, "%v: cost %d, expected %d (a = %v, b = %v)", opts, cost(c), expected, a, b, loc = loc) + } + return testing.expectf(t, cost(c) >= expected, "%v: cost %d below minimal %d (a = %v, b = %v)", opts, cost(c), expected, a, b, loc = loc) +} + +// Checks one pair under every option set in `PAIR_OPTIONS`. +check_pair :: proc(t: ^testing.T, a, b: []i32, loc := #caller_location) -> bool { + expected := minimal_cost(a, b) + for opts in PAIR_OPTIONS { + c := diff_exact(a, b, opts) + defer diff.changes_destroy(c, context.allocator) + check_valid(t, a, b, c, loc) or_return + check_cost(t, a, b, c, opts, expected, loc) or_return + } + return true +} + +// Writes the base-`base` digits of `code` into `out`. +decode :: proc(code, base: int, out: []i32) { + code := code + for &x in out { + x = i32(code % base) + code /= base + } +} + +// Every pair of sequences over `alphabet` symbols, each up to `max_len` long. +check_exhaustive :: proc(t: ^testing.T, alphabet, max_len: int, opts := diff.Options{algorithm = .Minimal}) { + a_buf := make([]i32, max_len) + defer delete(a_buf) + b_buf := make([]i32, max_len) + defer delete(b_buf) + + bits := make([]u64, diff.changes_size(max_len, max_len)) + defer delete(bits) + scratch := make([]byte, diff.scratch_size_interned(max_len, max_len, alphabet, opts)) + defer delete(scratch) + rows := make([]int, 2*(max_len + 1)) + defer delete(rows) + + for len_a in 0..=max_len { + for code_a in 0.. int { + n := 1 + for _ in 0.. []i32 { + s := make([]i32, length) + for &x in s { + x = i32(rand.int_max(alphabet)) + } + return s +} + +// Applies `edits` random insertions, deletions and replacements to `base`. +mutate :: proc(base: []i32, edits, alphabet: int) -> []i32 { + out := make([dynamic]i32, 0, len(base) + edits) + append(&out, ..base) + for _ in 0.. 0 { + ordered_remove(&out, rand.int_max(len(out))) + } + case 2: + if len(out) > 0 { + out[rand.int_max(len(out))] = i32(rand.int_max(alphabet)) + } + } + } + defer delete(out) + return slice.clone(out[:]) +} + + +/* + Tests +*/ + +@(test) +test_empty_and_identical :: proc(t: ^testing.T) { + x := []i32{1, 2, 3, 4} + y := []i32{5, 6, 7} + + check_pair(t, nil, nil) + check_pair(t, nil, x) + check_pair(t, x, nil) + check_pair(t, x, x) + check_pair(t, x, y) + + c := diff_exact(x, x) + defer diff.changes_destroy(c, context.allocator) + testing.expect_value(t, diff.stats(c), diff.Stats{equal = 4}) + + c_disjoint := diff_exact(x, y) + defer diff.changes_destroy(c_disjoint, context.allocator) + testing.expect_value(t, diff.stats(c_disjoint), diff.Stats{deleted = 4, inserted = 3}) +} + +// The example from Myers' paper: "ABCABBA" against "CBABAC" has a minimal diff of 5. +@(test) +test_myers_paper_example :: proc(t: ^testing.T) { + A, B, C :: 0, 1, 2 + a := []i32{A, B, C, A, B, B, A} + b := []i32{C, B, A, B, A, C} + + check_pair(t, a, b) + check_pair(t, b, a) + + c := diff_exact(a, b, {algorithm = .Minimal}) + defer diff.changes_destroy(c, context.allocator) + testing.expect_value(t, cost(c), 5) +} + +@(test) +test_minimal_exhaustive_binary :: proc(t: ^testing.T) { + check_exhaustive(t, 2, 8) +} + +@(test) +test_minimal_exhaustive_ternary :: proc(t: ^testing.T) { + check_exhaustive(t, 3, 5) +} + +// Small diffs never reach the default cap, so `.Myers` must still be minimal. +@(test) +test_myers_exhaustive_binary :: proc(t: ^testing.T) { + check_exhaustive(t, 2, 7, {algorithm = .Myers}) +} + +// Caps this small are hit on almost every split. +@(test) +test_cost_cap_exhaustive :: proc(t: ^testing.T) { + check_exhaustive(t, 2, 7, {algorithm = .Myers, max_cost = 1}) + check_exhaustive(t, 3, 4, {algorithm = .Myers, max_cost = 2}) +} + +/* + Long inputs with many sparse edits. Past 65k tokens the default cap rises to 512, which leaves room + for the snake heuristic after 256 edits; a cap of 64 is hit instead. +*/ +@(test) +test_cost_cap_heuristics :: proc(t: ^testing.T) { + rand.reset(t.seed) + for edits in ([?]int{800, 1600}) { + a := random_sequence(40_000, 100_000) + defer delete(a) + b := mutate(a, edits, 100_000) + defer delete(b) + + // Too big for `minimal_cost`; `.Minimal` is checked against it everywhere else + minimal := diff_exact(a, b, {algorithm = .Minimal}) + defer diff.changes_destroy(minimal, context.allocator) + check_valid(t, a, b, minimal) + expected := cost(minimal) + + for opts in ([?]diff.Options{{algorithm = .Myers}, {algorithm = .Myers, max_cost = 64}}) { + c := diff_exact(a, b, opts) + defer diff.changes_destroy(c, context.allocator) + check_valid(t, a, b, c) + check_cost(t, a, b, c, opts, expected) + } + } +} + +// Nothing in common, far past the cap: without it this would take O((n+m)^2) time. +@(test) +test_cost_cap_disjoint :: proc(t: ^testing.T) { + a := make([]i32, 20_000) + defer delete(a) + b := make([]i32, 20_000) + defer delete(b) + for i in 0.. bool { + i, j := 0, 0 + for { + a_end := i + for a_end < len(a) && diff.is_deleted(c, a_end) { + a_end += 1 + } + b_end := j + for b_end < len(b) && diff.is_inserted(c, b_end) { + b_end += 1 + } + can_slide := j < b_end && b_end < len(b) && b[j] == b[b_end] + testing.expectf(t, !can_slide || i < a_end, "%v: b[%d:%d] could slide down, with no deletions beside it (a = %v, b = %v)", opts, j, b_end, a, b, loc = loc) or_return + + if a_end == len(a) || b_end == len(b) { + return true + } + i, j = a_end + 1, b_end + 1 + } +} + + +/* + Tests +*/ + +@(test) +test_compact_exhaustive :: proc(t: ^testing.T) { + check_exhaustive(t, 2, 7, {algorithm = .Minimal, flags = {.Compact}}) + check_exhaustive(t, 2, 7, {algorithm = .Histogram, flags = {.Compact}}) +} + +// Histogram is left out. Diffing merged groups again can leave them anywhere. +@(test) +test_compact_settles_groups :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<300 { + alphabet := 1 + rand.int_max(5) + a := random_sequence(rand.int_max(200), alphabet) + defer delete(a) + b := mutate(a, rand.int_max(40), alphabet) + defer delete(b) + for algorithm in ([?]diff.Algorithm{.Myers, .Minimal, .Ratcliff_Obershelp}) { + for flags in ([?]diff.Option_Flags{{.Compact}, {.Prune, .Compact}}) { + opts := diff.Options{algorithm = algorithm, flags = flags} + c := diff_exact(a, b, opts) + defer diff.changes_destroy(c, context.allocator) + if !check_valid(t, a, b, c) || !check_settled(t, a, b, c, opts) { + return + } + } + } + } +} + +// A group that can slide is left as far down as it goes. +@(test) +test_compact_slides_down :: proc(t: ^testing.T) { + for algorithm in diff.Algorithm { + // Inserting `2 3` after `1 2 3` could be at either copy: it goes to the last + { + a := []i32{1, 2, 3} + b := []i32{1, 2, 3, 2, 3} + c := diff_exact(a, b, {algorithm = algorithm, flags = {.Compact}}) + defer diff.changes_destroy(c, context.allocator) + testing.expectf(t, diff.is_inserted(c, 3) && diff.is_inserted(c, 4) && !diff.is_inserted(c, 1), "%v: the insertion did not slide down", algorithm) + } + + // Deleting one of two `2`s deletes the second + { + a := []i32{1, 2, 2, 3} + b := []i32{1, 2, 3} + c := diff_exact(a, b, {algorithm = algorithm, flags = {.Compact}}) + defer diff.changes_destroy(c, context.allocator) + testing.expectf(t, diff.is_deleted(c, 2) && !diff.is_deleted(c, 1), "%v: the deletion did not slide down", algorithm) + } + } +} + +// Sliding never changes how many tokens change, except where Histogram diffs merged groups again. +@(test) +test_compact_keeps_cost :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<200 { + alphabet := 1 + rand.int_max(6) + a := random_sequence(rand.int_max(150), alphabet) + defer delete(a) + b := mutate(a, rand.int_max(30), alphabet) + defer delete(b) + for algorithm in diff.Algorithm { + plain := diff_exact(a, b, {algorithm = algorithm}) + defer diff.changes_destroy(plain, context.allocator) + compacted := diff_exact(a, b, {algorithm = algorithm, flags = {.Compact}}) + defer diff.changes_destroy(compacted, context.allocator) + + if algorithm == .Histogram { + testing.expectf(t, cost(compacted) <= cost(plain), "Histogram: compacting grew the diff from %d to %d", cost(plain), cost(compacted)) + } else { + testing.expectf(t, cost(compacted) == cost(plain), "%v: compacting changed the cost from %d to %d", algorithm, cost(plain), cost(compacted)) + } + } + } +} + +@(test) +test_compact_text :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<100 { + a := random_text(rand.int_max(80)) + b := mutate_lines(a, rand.int_max(20)) + if !check_text_matches_lines(t, a, b, {}, {flags = {.Prune, .Compact}}) { + return + } + if !check_unified(t, a, b, 3) { + return + } + } +} diff --git a/tests/core/diff/test_core_diff_edit.odin b/tests/core/diff/test_core_diff_edit.odin new file mode 100644 index 000000000..515cb14d3 --- /dev/null +++ b/tests/core/diff/test_core_diff_edit.odin @@ -0,0 +1,373 @@ +package test_core_diff + +import "core:diff" +import "core:math/rand" +import "core:slice" +import "core:testing" + +/* + Harness +*/ + +collect_edits :: proc(c: diff.Changes) -> [dynamic]diff.Edit { + edits: [dynamic]diff.Edit + it := diff.edit_iterator_make(c) + for e in diff.edit_iterate(&it) { + append(&edits, e) + } + return edits +} + +collect_hunks :: proc(c: diff.Changes, context_len: int) -> [dynamic]diff.Hunk { + hunks: [dynamic]diff.Hunk + it := diff.hunk_iterator_make(c, context_len) + for h in diff.hunk_iterate(&it) { + append(&hunks, h) + } + return hunks +} + +/* + Checks the edits of a valid diff of `a` and `b`: + - they cover both sides in order, with no gaps, overlaps or empty runs + - a `.Delete` has an empty `b` range, an `.Insert` an empty `a` range, and an `.Equal` the same length on both sides + - `.Equal` runs hold equal tokens, and the tokens of `.Delete` and `.Insert` runs are marked + - neighbours never share an op, and an `.Insert` is never followed by a `.Delete` + - `b` can be rebuilt from `a` and the edits + - `edit_count`, `edits_fill` and `edits_make` agree with the iterator +*/ +check_edits :: proc(t: ^testing.T, a, b: []i32, c: diff.Changes, loc := #caller_location) -> bool { + edits := collect_edits(c) + defer delete(edits) + + rebuilt: [dynamic]i32 + defer delete(rebuilt) + + i, j := 0, 0 + for e, n in edits { + testing.expectf(t, e.a_lo == i && e.b_lo == j, "edit %d %v does not start at a[%d], b[%d]", n, e, i, j, loc = loc) or_return + testing.expectf(t, e.a_hi > e.a_lo || e.b_hi > e.b_lo, "edit %d %v is empty", n, e, loc = loc) or_return + + switch e.op { + case .Equal: + testing.expectf(t, e.a_hi - e.a_lo == e.b_hi - e.b_lo, "edit %d %v has sides of different lengths", n, e, loc = loc) or_return + for k in 0.. 0 { + prev := edits[n-1] + testing.expectf(t, prev.op != e.op, "edits %d and %d are both %v", n-1, n, e.op, loc = loc) or_return + testing.expectf(t, !(prev.op == .Insert && e.op == .Delete), "edit %d is a .Delete after an .Insert", n, loc = loc) or_return + } + i, j = e.a_hi, e.b_hi + } + testing.expectf(t, i == len(a) && j == len(b), "edits end at a[%d], b[%d], not a[%d], b[%d]", i, j, len(a), len(b), loc = loc) or_return + testing.expect(t, slice.equal(rebuilt[:], b), "b could not be rebuilt from the edits", loc = loc) or_return + + testing.expect_value(t, diff.edit_count(c), len(edits), loc = loc) or_return + + buf := make([]diff.Edit, len(edits)) + defer delete(buf) + testing.expect(t, slice.equal(diff.edits_fill(c, buf), edits[:]), "edits_fill disagrees with the iterator", loc = loc) or_return + + made := must(diff.edits_make(c, context.allocator)) + defer delete(made) + testing.expect(t, slice.equal(made, edits[:]), "edits_make disagrees with the iterator", loc = loc) or_return + return true +} + +// Hunks worked out from the edit list rather than the bits. +reference_hunks :: proc(c: diff.Changes, context_len: int) -> [dynamic]diff.Hunk { + edits := collect_edits(c) + defer delete(edits) + + // Each change is a `.Delete`, an `.Insert`, or a `.Delete` then an `.Insert` + changes: [dynamic]diff.Hunk + defer delete(changes) + for e in edits { + if e.op == .Equal { + continue + } + if len(changes) > 0 && changes[len(changes)-1].a_hi == e.a_lo && changes[len(changes)-1].b_hi == e.b_lo { + last := &changes[len(changes)-1] + last.a_hi, last.b_hi = e.a_hi, e.b_hi + } else { + append(&changes, diff.Hunk{e.a_lo, e.a_hi, e.b_lo, e.b_hi}) + } + } + + hunks: [dynamic]diff.Hunk + for ch, k in changes { + gap_before := k == 0 ? ch.a_lo : ch.a_lo - changes[k-1].a_hi + gap_after := k == len(changes)-1 ? c.n - ch.a_hi : changes[k+1].a_lo - ch.a_hi + if k == 0 || gap_before > 2*context_len { + lead := min(context_len, gap_before) + append(&hunks, diff.Hunk{ch.a_lo - lead, 0, ch.b_lo - lead, 0}) + } + trail := min(context_len, gap_after) + h := &hunks[len(hunks)-1] + h.a_hi, h.b_hi = ch.a_hi + trail, ch.b_hi + trail + } + return hunks +} + +/* + Checks the hunks of a valid diff against `reference_hunks`, and that iterating the edits of each hunk + gives its context, at most `context_len` tokens on each end, around at least one change. +*/ +check_hunks :: proc(t: ^testing.T, c: diff.Changes, context_len: int, loc := #caller_location) -> bool { + hunks := collect_hunks(c, context_len) + defer delete(hunks) + expected := reference_hunks(c, context_len) + defer delete(expected) + testing.expectf(t, slice.equal(hunks[:], expected[:]), "context %d: hunks %v, expected %v", context_len, hunks, expected, loc = loc) or_return + + for h in hunks { + edits: [dynamic]diff.Edit + defer delete(edits) + it := diff.edit_iterator_make_hunk(c, h) + for e in diff.edit_iterate(&it) { + append(&edits, e) + } + + testing.expectf(t, len(edits) > 0, "hunk %v has no edits", h, loc = loc) or_return + first, last := edits[0], edits[len(edits)-1] + testing.expectf(t, first.a_lo == h.a_lo && first.b_lo == h.b_lo, "hunk %v starts with %v", h, first, loc = loc) or_return + testing.expectf(t, last.a_hi == h.a_hi && last.b_hi == h.b_hi, "hunk %v ends with %v", h, last, loc = loc) or_return + if first.op == .Equal { + testing.expectf(t, first.a_hi - first.a_lo <= context_len, "hunk %v leads with %v", h, first, loc = loc) or_return + } + if last.op == .Equal { + testing.expectf(t, last.a_hi - last.a_lo <= context_len, "hunk %v trails with %v", h, last, loc = loc) or_return + } + testing.expectf(t, len(edits) > 1 || first.op != .Equal, "hunk %v holds no change", h, loc = loc) or_return + } + return true +} + +@(rodata) +HUNK_CONTEXTS := [?]int{0, 1, 2, 3, 5} + +check_views :: proc(t: ^testing.T, a, b: []i32, opts := diff.Options{}, loc := #caller_location) -> bool { + c := diff_exact(a, b, opts) + defer diff.changes_destroy(c, context.allocator) + check_valid(t, a, b, c, loc) or_return + check_edits(t, a, b, c, loc) or_return + for context_len in HUNK_CONTEXTS { + check_hunks(t, c, context_len, loc) or_return + } + return true +} + +// `a` is `0.. (a, b: []i32) { + a = make([]i32, n) + b = make([]i32, n) + for i in 0.. string { + d := must(diff.diff_text(a, b, context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + + scratch := make([]byte, diff.emphasis_scratch_size(d, emphasis), allocator) + return must(diff.unified(d, allocator, context_len = context_len, style = style, emphasis = emphasis, scratch = scratch)) +} + +expect_emphasis :: proc(t: ^testing.T, a, b: string, emphasis: diff.Emphasis, expected: string, style := MARKERS, loc := #caller_location) { + s := emphasised(a, b, emphasis, style) + testing.expectf(t, s == expected, "%v: got:\n%s\nexpected:\n%s", emphasis, s, expected, loc = loc) +} + +/* + Emphasis only adds markers: taking them out must give the plain output, and every way + of writing the output must agree on its size. +*/ +check_emphasis :: proc(t: ^testing.T, a, b: string, emphasis: diff.Emphasis, context_len: int, loc := #caller_location) -> bool { + d := must(diff.diff_text(a, b, context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + + scratch := make([]byte, diff.emphasis_scratch_size(d, emphasis), context.temp_allocator) + marked := must(diff.unified(d, context.temp_allocator, context_len = context_len, style = MARKERS, emphasis = emphasis, scratch = scratch)) + plain := must(diff.unified(d, context.temp_allocator, context_len = context_len)) + + size := diff.unified_size(d, context_len = context_len, style = MARKERS, emphasis = emphasis, scratch = scratch) + testing.expect_value(t, size, len(marked), loc = loc) or_return + + grown := must(diff.unified(d, context.temp_allocator, context_len = context_len, style = MARKERS, emphasis = emphasis, scratch = scratch, sizing = .Grow)) + testing.expect_value(t, grown, marked, loc = loc) or_return + + stripped, _ := strings.remove_all(marked, "[", context.temp_allocator) + stripped, _ = strings.remove_all(stripped, "]", context.temp_allocator) + return testing.expectf(t, stripped == plain, "%v: without its markers:\n%s\nis not:\n%s", emphasis, stripped, plain, loc = loc) +} + + +/* + Tests +*/ + +@(test) +test_emphasis_prefix_suffix :: proc(t: ^testing.T) { + expect_emphasis(t, "x\nhello world\ny\n", "x\nhello there\ny\n", .Prefix_Suffix, + "--- a\n" + + "+++ b\n" + + "@@ -1,3 +1,3 @@\n" + + " x\n" + + "-hello [world]\n" + + "+hello [there]\n" + + " y\n") + + // Something inserted in the middle of a line: only the new part is marked + expect_emphasis(t, "abc\n", "abXc\n", .Prefix_Suffix, + "--- a\n" + + "+++ b\n" + + "@@ -1 +1 @@\n" + + "-abc\n" + + "+ab[X]c\n") + + // Pairs need as many lines on each side + expect_emphasis(t, "ab\ncd\n", "aX\n", .Prefix_Suffix, + "--- a\n" + + "+++ b\n" + + "@@ -1,2 +1 @@\n" + + "-ab\n" + + "-cd\n" + + "+aX\n") + + // Nothing in common, nothing marked + expect_emphasis(t, "abc\n", "xyz\n", .Prefix_Suffix, + "--- a\n" + + "+++ b\n" + + "@@ -1 +1 @@\n" + + "-abc\n" + + "+xyz\n") +} + +// A common prefix or suffix never splits a rune. +@(test) +test_emphasis_runes :: proc(t: ^testing.T) { + expect_emphasis(t, "naïve\n", "naive\n", .Prefix_Suffix, + "--- a\n" + + "+++ b\n" + + "@@ -1 +1 @@\n" + + "-na[ï]ve\n" + + "+na[i]ve\n") + + // "é" and "è" share their first byte, which is not a rune of its own + expect_emphasis(t, "aé\n", "aè\n", .Prefix_Suffix, + "--- a\n" + + "+++ b\n" + + "@@ -1 +1 @@\n" + + "-a[é]\n" + + "+a[è]\n") +} + +@(test) +test_emphasis_words :: proc(t: ^testing.T) { + expect_emphasis(t, "the quick brown fox\n", "the slow brown fox\n", .Words, + "--- a\n" + + "+++ b\n" + + "@@ -1 +1 @@\n" + + "-the [quick] brown fox\n" + + "+the [slow] brown fox\n") + + // Words are diffed across the whole block, whatever the line counts + expect_emphasis(t, "foo bar\nbaz qux\n", "foo BAR\nbaz qux quux\n", .Words, + "--- a\n" + + "+++ b\n" + + "@@ -1,2 +1,2 @@\n" + + "-foo [bar]\n" + + "-baz qux\n" + + "+foo [BAR]\n" + + "+baz qux[ quux]\n") + + // A changed word of whitespace can hold a line break, which is never shown + expect_emphasis(t, "one two\n", "one\ntwo\n", .Words, + "--- a\n" + + "+++ b\n" + + "@@ -1 +1,2 @@\n" + + "-one[ ]two\n" + + "+one\n" + + "+two\n") + + // Nothing in common, nothing marked + expect_emphasis(t, "abc\n", "xyz\n", .Words, + "--- a\n" + + "+++ b\n" + + "@@ -1 +1 @@\n" + + "-abc\n" + + "+xyz\n") +} + +@(test) +test_emphasis_ansi :: proc(t: ^testing.T) { + expect_emphasis(t, "hello world!\n", "hello there!\n", .Prefix_Suffix, style = diff.STYLE_ANSI, expected = + "\x1b[1m--- a\x1b[0m\n" + + "\x1b[1m+++ b\x1b[0m\n" + + "\x1b[36m@@ -1 +1 @@\x1b[0m\n" + + "\x1b[31m-hello \x1b[31;7mworld\x1b[0m\x1b[31m!\x1b[0m\n" + + "\x1b[32m+hello \x1b[32;7mthere\x1b[0m\x1b[32m!\x1b[0m\n") +} + +// Without emphasis colours there is nothing to show, meaning no scratch is needed either. +@(test) +test_emphasis_without_colours :: proc(t: ^testing.T) { + d := must(diff.diff_text("the quick fox\n", "the slow fox\n", context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + + plain := must(diff.unified(d, context.allocator)) + defer delete(plain) + for emphasis in diff.Emphasis { + s := must(diff.unified(d, context.allocator, emphasis = emphasis)) + defer delete(s) + testing.expectf(t, s == plain, "%v without colours changed the output", emphasis) + } +} + +@(test) +test_emphasis_scratch_size :: proc(t: ^testing.T) { + d := must(diff.diff_text("same\nthe quick fox\nsame\nab\n", "same\nthe slow fox\nsame\nxy\nzz\n", context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + + testing.expect_value(t, diff.emphasis_scratch_size(d, .None), 0) + testing.expect_value(t, diff.emphasis_scratch_size(d, .Prefix_Suffix), 0) + + // Exactly the size asked for, at every alignment + size := diff.emphasis_scratch_size(d, .Words) + testing.expect(t, size > 0, "a diff with changes asked for no scratch") + buf := make([]byte, size + 16) + defer delete(buf) + + expected := emphasised(d.a, d.b, .Words) + for offset in 0..<16 { + s := must(diff.unified(d, context.temp_allocator, style = MARKERS, emphasis = .Words, scratch = buf[offset:][:size])) + testing.expectf(t, s == expected, "offset %d gave:\n%s", offset, s) + } + + // Only insertions, or no changes at all: nothing to diff words of + only_inserts := must(diff.diff_text("a\n", "a\nb\n", context.allocator, context.allocator)) + defer diff.text_diff_destroy(only_inserts, context.allocator) + testing.expect_value(t, diff.emphasis_scratch_size(only_inserts, .Words), 0) +} + +@(test) +test_emphasis_random :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<150 { + a := random_text(rand.int_max(60)) + b := mutate_lines(a, rand.int_max(15)) + for emphasis in ([?]diff.Emphasis{.Prefix_Suffix, .Words}) { + for context_len in ([?]int{0, 3}) { + if !check_emphasis(t, a, b, emphasis, context_len) { + return + } + } + } + } +} diff --git a/tests/core/diff/test_core_diff_generic.odin b/tests/core/diff/test_core_diff_generic.odin new file mode 100644 index 000000000..51b184d1a --- /dev/null +++ b/tests/core/diff/test_core_diff_generic.odin @@ -0,0 +1,325 @@ +package test_core_diff + +import "base:runtime" +import "core:diff" +import "core:math/rand" +import "core:mem" +import "core:slice" +import "core:strings" +import "core:testing" + +/* + Harness +*/ + +// IDs by first appearance in `a` then `b`, worked out with a map rather than `core:diff`'s own table. +ids_of :: proc(a, b: []$T) -> (ia, ib: []i32, id_count: int) { + seen := make(map[T]i32) + defer delete(seen) + + ia = make([]i32, len(a)) + ib = make([]i32, len(b)) + for x, k in a { + if x not_in seen { + seen[x] = i32(len(seen)) + } + ia[k] = seen[x] + } + for x, k in b { + if x not_in seen { + seen[x] = i32(len(seen)) + } + ib[k] = seen[x] + } + return ia, ib, len(seen) +} + +// The algorithms only look at which tokens are equal, so interning must not change the result by a single bit. +check_matches_interned :: proc(t: ^testing.T, a, b: []$T, c: diff.Changes, opts: diff.Options, loc := #caller_location) -> bool { + ia, ib, id_count := ids_of(a, b) + defer delete(ia) + defer delete(ib) + + check_valid(t, ia, ib, c, loc) or_return + + expected := must(diff.diff_interned(ia, ib, id_count, context.allocator, context.allocator, opts = opts)) + defer diff.changes_destroy(expected, context.allocator) + return testing.expectf(t, slice.equal(c.bits, expected.bits), "%v: result differs from diffing the interned IDs", opts, loc = loc) +} + +@(rodata) +WORDS := [?]string{"the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog", "", "fox\x00"} + +random_words :: proc(length, vocabulary: int) -> []string { + s := make([]string, length) + for &x in s { + x = WORDS[rand.int_max(min(vocabulary, len(WORDS)))] + } + return s +} + +Counting_Allocator :: struct { + backing: runtime.Allocator, + allocs: int, + frees: int, + fail: bool, // every allocation fails +} + +counting_allocator :: proc(ca: ^Counting_Allocator) -> runtime.Allocator { + return {counting_allocator_proc, ca} +} + +counting_allocator_proc :: proc(data: rawptr, mode: runtime.Allocator_Mode, size, alignment: int, old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, runtime.Allocator_Error) { + ca := (^Counting_Allocator)(data) + #partial switch mode { + case .Alloc, .Alloc_Non_Zeroed: + if ca.fail { + return nil, .Out_Of_Memory + } + ca.allocs += 1 + case .Free: + ca.frees += 1 + } + return ca.backing.procedure(ca.backing.data, mode, size, alignment, old_memory, old_size, loc) +} + +Folded :: struct { + equals: int, + hashes: int, +} + +// Case-insensitive equality, counting its calls through `user_data`. +folded_equal :: proc(x, y: string, user_data: rawptr) -> bool { + (^Folded)(user_data).equals += 1 + return strings.equal_fold(x, y) +} + +folded_hash :: proc(x: string, user_data: rawptr) -> u64 { + (^Folded)(user_data).hashes += 1 + h := u64(14695981039346656037) + for c in transmute([]byte)x { + h = (h ~ u64(c | 0x20)) * 1099511628211 + } + return h +} + +constant_hash :: proc(x: string, _: rawptr) -> u64 { + return 7 +} + +string_equal :: proc(x, y: string, _: rawptr) -> bool { + return x == y +} + + +/* + Tests +*/ + +@(test) +test_generic_ints :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<100 { + alphabet := 1 + rand.int_max(12) + a32 := random_sequence(rand.int_max(200), alphabet) + defer delete(a32) + b32 := mutate(a32, rand.int_max(40), alphabet) + defer delete(b32) + + // Widen and spread the values, so they are nothing like dense IDs + a := make([]int, len(a32)) + defer delete(a) + b := make([]int, len(b32)) + defer delete(b) + for x, k in a32 { + a[k] = int(x)*1_000_003 - 500 + } + for x, k in b32 { + b[k] = int(x)*1_000_003 - 500 + } + + for opts in PAIR_OPTIONS { + c := must(diff.diff(a, b, context.allocator, context.allocator, opts = opts)) + defer diff.changes_destroy(c, context.allocator) + if !check_matches_interned(t, a, b, c, opts) { + return + } + } + } +} + +@(test) +test_generic_strings :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<100 { + vocabulary := 1 + rand.int_max(len(WORDS)) + a := random_words(rand.int_max(150), vocabulary) + defer delete(a) + b := random_words(rand.int_max(150), vocabulary) + defer delete(b) + + c := must(diff.diff(a, b, context.allocator, context.allocator)) + defer diff.changes_destroy(c, context.allocator) + if !check_matches_interned(t, a, b, c, {}) { + return + } + } +} + +@(test) +test_generic_structs :: proc(t: ^testing.T) { + Token :: struct { + kind: u8, + text: string, + line: i32, + } + a := []Token{{1, "x", 1}, {2, "y", 1}, {1, "x", 2}, {3, "", 0}, {2, "y", 1}} + b := []Token{{2, "y", 1}, {1, "x", 1}, {3, "", 0}, {1, "z", 1}, {2, "y", 1}} + + c := must(diff.diff(a, b, context.allocator, context.allocator)) + defer diff.changes_destroy(c, context.allocator) + check_matches_interned(t, a, b, c, {}) +} + +@(test) +test_generic_empty :: proc(t: ^testing.T) { + x := []string{"a", "b"} + + c := must(diff.diff([]string{}, []string{}, context.allocator, context.allocator)) + defer diff.changes_destroy(c, context.allocator) + testing.expect_value(t, diff.stats(c), diff.Stats{}) + + c_insert := must(diff.diff([]string{}, x, context.allocator, context.allocator)) + defer diff.changes_destroy(c_insert, context.allocator) + testing.expect_value(t, diff.stats(c_insert), diff.Stats{inserted = 2}) + + c_delete := must(diff.diff(x, []string{}, context.allocator, context.allocator)) + defer diff.changes_destroy(c_delete, context.allocator) + testing.expect_value(t, diff.stats(c_delete), diff.Stats{deleted = 2}) +} + +@(test) +test_diff_by_custom_equality :: proc(t: ^testing.T) { + a := []string{"Hello", "World", "foo", "BAR", "baz"} + b := []string{"hello", "world", "FOO", "qux", "bar", "Baz"} + + folded: Folded + c := must(diff.diff_by(a, b, folded_equal, folded_hash, &folded, context.allocator, context.allocator)) + defer diff.changes_destroy(c, context.allocator) + + testing.expect_value(t, folded.hashes, len(a) + len(b)) + testing.expect(t, folded.equals > 0, "equal was never called through user_data") + + // The same as diffing the lowered strings + lower_a := make([]string, len(a)) + defer delete(lower_a) + lower_b := make([]string, len(b)) + defer delete(lower_b) + for s, k in a { + lower_a[k] = strings.to_lower(s, context.temp_allocator) + } + for s, k in b { + lower_b[k] = strings.to_lower(s, context.temp_allocator) + } + expected := must(diff.diff(lower_a, lower_b, context.allocator, context.allocator)) + defer diff.changes_destroy(expected, context.allocator) + testing.expect(t, slice.equal(c.bits, expected.bits), "case-insensitive diff differs from diffing the lowered strings") + testing.expect_value(t, diff.stats(c), diff.Stats{equal = 5, inserted = 1}) +} + +// Every token hashing the same must still be correct: `equal` alone tells the tokens apart. +@(test) +test_diff_by_hash_collisions :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<50 { + a := random_words(rand.int_max(80), len(WORDS)) + defer delete(a) + b := random_words(rand.int_max(80), len(WORDS)) + defer delete(b) + + c := must(diff.diff_by(a, b, string_equal, constant_hash, nil, context.allocator, context.allocator)) + defer diff.changes_destroy(c, context.allocator) + if !check_matches_interned(t, a, b, c, {}) { + return + } + } +} + +// The convenience procedures make exactly one allocation from each allocator, and free only the scratch. +@(test) +test_allocations :: proc(t: ^testing.T) { + a := []string{"a", "b", "c", "d"} + b := []string{"a", "c", "d", "e"} + + result := Counting_Allocator{backing = context.allocator} + transient := Counting_Allocator{backing = context.allocator} + + c := must(diff.diff(a, b, counting_allocator(&result), counting_allocator(&transient))) + testing.expect_value(t, result.allocs, 1) + testing.expect_value(t, result.frees, 0) + testing.expect_value(t, transient.allocs, 1) + testing.expect_value(t, transient.frees, 1) + diff.changes_destroy(c, counting_allocator(&result)) + testing.expect_value(t, result.frees, 1) +} + +// When the scratch cannot be had, the result is freed again; the test runner reports any leak. +@(test) +test_allocation_failure :: proc(t: ^testing.T) { + a := []string{"a", "b"} + b := []string{"b", "c"} + + result := Counting_Allocator{backing = context.allocator} + transient := Counting_Allocator{backing = context.allocator, fail = true} + + c, err := diff.diff(a, b, counting_allocator(&result), counting_allocator(&transient)) + testing.expect_value(t, err, runtime.Allocator_Error.Out_Of_Memory) + testing.expect_value(t, len(c.bits), 0) + testing.expect_value(t, result.allocs, 1) + testing.expect_value(t, result.frees, 1) +} + +// One fixed buffer exactly big enough for the result and the scratch, used as both allocators. +@(test) +test_single_arena :: proc(t: ^testing.T) { + rand.reset(t.seed) + a := random_words(500, len(WORDS)) + defer delete(a) + b := random_words(450, len(WORDS)) + defer delete(b) + + needed := diff.changes_size(len(a), len(b))*size_of(u64) + diff.scratch_size(len(a), len(b)) + buf := make([]byte, needed + 8) // the result is aligned to 8 + defer delete(buf) + + arena: mem.Arena + mem.arena_init(&arena, buf) + arena_allocator := mem.arena_allocator(&arena) + + c, err := diff.diff(a, b, arena_allocator, arena_allocator) + testing.expect_value(t, err, nil) + check_matches_interned(t, a, b, c, {}) +} + +// Scratch of exactly `scratch_size` bytes must work at every alignment. +@(test) +test_generic_scratch_alignment :: proc(t: ^testing.T) { + rand.reset(t.seed) + a := random_words(300, len(WORDS)) + defer delete(a) + b := random_words(320, len(WORDS)) + defer delete(b) + + size := diff.scratch_size(len(a), len(b)) + buf := make([]byte, size + 16) + defer delete(buf) + + for offset in 0..<16 { + c := must(diff.changes_make(len(a), len(b), context.allocator)) + defer diff.changes_destroy(c, context.allocator) + diff.diff_buffer(a, b, c, buf[offset:][:size]) + if !check_matches_interned(t, a, b, c, {}) { + return + } + } +} diff --git a/tests/core/diff/test_core_diff_histogram.odin b/tests/core/diff/test_core_diff_histogram.odin new file mode 100644 index 000000000..75a3a9792 --- /dev/null +++ b/tests/core/diff/test_core_diff_histogram.odin @@ -0,0 +1,179 @@ +package test_core_diff + +import "core:diff" +import "core:math/rand" +import "core:slice" +import "core:strings" +import "core:testing" + +/* + Harness +*/ + +// Histogram must give exactly what Myers with `.Prune` gives under the same cap. +check_histogram_falls_back :: proc(t: ^testing.T, a, b: []i32, loc := #caller_location) -> bool { + for max_cost in ([?]int{0, 1, 3}) { + histogram := diff_exact(a, b, {algorithm = .Histogram, max_cost = max_cost}) + defer diff.changes_destroy(histogram, context.allocator) + myers := diff_exact(a, b, {algorithm = .Myers, flags = {.Prune}, max_cost = max_cost}) + defer diff.changes_destroy(myers, context.allocator) + + check_valid(t, a, b, histogram, loc) or_return + testing.expectf(t, slice.equal(histogram.bits, myers.bits), "max_cost %d: the fallback differs from Myers with .Prune (a = %v, b = %v)", max_cost, a, b, loc = loc) or_return + } + return true +} + + +/* + Tests +*/ + +@(test) +test_histogram_exhaustive :: proc(t: ^testing.T) { + check_exhaustive(t, 2, 7, {algorithm = .Histogram}) + check_exhaustive(t, 3, 5, {algorithm = .Histogram}) + check_exhaustive(t, 4, 4, {algorithm = .Histogram}) +} + +/* + When every token in common occurs more than 64 times in `a`, Histogram has no anchor. It falls back to Myers + with `.Prune`, under the same cap. +*/ +@(test) +test_histogram_fallback :: proc(t: ^testing.T) { + { + a := make([dynamic]i32) + defer delete(a) + b := make([dynamic]i32) + defer delete(b) + + // No common prefix or suffix, and only 0s and 1s, 70 of each, in common + append(&a, 7) + append(&b, 8) + for i in 0..<70 { + append(&a, 0) + append(&a, 1) + append(&b, i32(1 - (i & 1))) + append(&b, i32(i & 1)) + } + append(&a, 9) + check_histogram_falls_back(t, a[:], b[:]) + } + + // Two frequent tokens, among tokens found on one side only. `.Prune` sets those aside. + rand.reset(t.seed) + for _ in 0..<20 { + a := frequent_and_unique(300 + rand.int_max(300), 2, 1_000, 3) + defer delete(a) + b := frequent_and_unique(300 + rand.int_max(300), 2, 100_000, 3) + defer delete(b) + if !(slice.count(a, 0) > 64 && slice.count(a, 1) > 64) { + continue + } + if !check_histogram_falls_back(t, a, b) { + return + } + } +} + +/* + Histogram only anchors on tokens occurring 65 times or fewer in `a`. Here every token in common occurs more often, + but one, found once on each side. It is the only anchor. It is always kept. +*/ +@(test) +test_histogram_one_anchor :: proc(t: ^testing.T) { + RARE :: 1_000_000 + + rand.reset(t.seed) + for _ in 0..<50 { + frequent := 1 + rand.int_max(3) + x := frequent_and_unique(150*frequent + rand.int_max(200), frequent, 1_000, 3 + rand.int_max(4)) + defer delete(x) + y := frequent_and_unique(rand.int_max(300), frequent, 100_000, 2 + rand.int_max(4)) + defer delete(y) + + too_rare := false + for f in 0.. string { + sb := strings.builder_make(context.temp_allocator) + strings.write_string(&sb, "func() {\n") + code_block(&sb, 1) + strings.write_string(&sb, "}\n\n") + return strings.to_string(sb) +} + +// Two texts of whole functions, the second with some inserted, removed or repeated. +code_pair :: proc() -> (a, b: string) { + functions := make([dynamic]string, context.temp_allocator) + for _ in 0..<3 + rand.int_max(8) { + append(&functions, code_function()) + } + a = strings.concatenate(functions[:], context.temp_allocator) + + for _ in 0..<1 + rand.int_max(3) { + switch rand.int_max(3) { + case 0: + inject_at(&functions, rand.int_max(len(functions) + 1), code_function()) + case 1: + if len(functions) > 1 { + ordered_remove(&functions, rand.int_max(len(functions))) + } + case 2: + i := rand.int_max(len(functions)) + inject_at(&functions, i, functions[i]) + } + } + b = strings.concatenate(functions[:], context.temp_allocator) + return +} + +// The heuristic only slides groups. The cost must stay as it was without `.Compact`, except where Histogram diffs merged groups again. +check_indent_keeps_cost :: proc(t: ^testing.T, a, b: string, algorithm: diff.Algorithm, loc := #caller_location) -> bool { + plain := must(diff.diff_text(a, b, context.allocator, context.allocator, opts = {algorithm = algorithm})) + defer diff.text_diff_destroy(plain, context.allocator) + d := must(diff.diff_text(a, b, context.allocator, context.allocator, opts = {algorithm = algorithm, flags = {.Compact, .Indent_Heuristic}})) + defer diff.text_diff_destroy(d, context.allocator) + + check_spans(t, d, true, loc) or_return + if algorithm == .Histogram { + return testing.expectf(t, cost(d.changes) <= cost(plain.changes), "Histogram: the heuristic grew the diff from %d to %d", cost(plain.changes), cost(d.changes), loc = loc) + } + return testing.expectf(t, cost(d.changes) == cost(plain.changes), "%v: the heuristic changed the cost from %d to %d", algorithm, cost(plain.changes), cost(d.changes), loc = loc) +} + +// Adding `.Indent_Heuristic` to `opts` must change nothing. +check_indent_changes_nothing :: proc(t: ^testing.T, a, b: string, opts: diff.Options, loc := #caller_location) -> bool { + plain := must(diff.diff_text(a, b, context.allocator, context.allocator, opts = opts)) + defer diff.text_diff_destroy(plain, context.allocator) + d := must(diff.diff_text(a, b, context.allocator, context.allocator, opts = {algorithm = opts.algorithm, flags = opts.flags + {.Indent_Heuristic}})) + defer diff.text_diff_destroy(d, context.allocator) + return testing.expectf(t, slice.equal(d.changes.bits, plain.changes.bits), "%v: the heuristic moved a group (a = %q, b = %q)", opts, a, b, loc = loc) +} + + +/* + Tests +*/ + +@(test) +test_indent_keeps_cost :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<100 { + a, b := code_pair() + for algorithm in diff.Algorithm { + if !check_indent_keeps_cost(t, a, b, algorithm) { + return + } + } + } +} + +/* + With every line indented the same and none blank, each position the heuristic weighs scores the same. Ties go to + the lowest, where `.Compact` alone leaves a group. Unique first and last lines keep groups off the ends of the + text, where positions score worse. +*/ +@(test) +test_indent_uniform :: proc(t: ^testing.T) { + @(require_results) + uniform_text :: proc(tokens: []i32, indent: string) -> string { + sb := strings.builder_make(context.temp_allocator) + fmt.sbprintf(&sb, "%sfirst\n", indent) + for x in tokens { + fmt.sbprintf(&sb, "%s%c\n", indent, 'a' + rune(x)) + } + fmt.sbprintf(&sb, "%slast\n", indent) + return strings.to_string(sb) + } + + + rand.reset(t.seed) + for _ in 0..<200 { + indent := rand.choice([]string{"", "\t", " "}) + alphabet := 1 + rand.int_max(4) + x := random_sequence(rand.int_max(60), alphabet) + defer delete(x) + y := mutate(x, rand.int_max(15), alphabet) + defer delete(y) + a := uniform_text(x, indent) + b := uniform_text(y, indent) + + for algorithm in diff.Algorithm { + for flags in ([?]diff.Option_Flags{{.Compact}, {.Prune, .Compact}}) { + if !check_indent_changes_nothing(t, a, b, {algorithm = algorithm, flags = flags}) { + return + } + } + } + } +} + +/* + A pair where git's indent heuristic changes the output. The expected text is what + `git diff --no-index` gives, which is Myers with `.Prune`, `.Compact` and `.Indent_Heuristic`. +*/ +@(test) +test_indent_matches_git :: proc(t: ^testing.T) { + a := "func() {\n\treturn;\n\n\tx1();\n}\n\nfunc() {\n\treturn;\n\n\tx2();\n}\n\nfunc() {\n\n \n \n \n}\n\nfunc() {\n\tx3();\n\tx1();\n}\n\nfunc() {\n\treturn;\n\n}\n\n" + b := "func() {\n\treturn;\n\n\tx2();\n}\n\nfunc() {\n\tx3();\n\tx1();\n}\n\nfunc() {\n\treturn;\n\n}\n\n" + + d := must(diff.diff_text(a, b, context.allocator, context.allocator, opts = {flags = {.Prune, .Compact, .Indent_Heuristic}})) + defer diff.text_diff_destroy(d, context.allocator) + s := must(diff.unified(d, context.temp_allocator)) + expected := + "--- a\n" + + "+++ b\n" + + "@@ -1,20 +1,7 @@\n" + + "-func() {\n" + + "-\treturn;\n" + + "-\n" + + "-\tx1();\n" + + "-}\n" + + "-\n" + + " func() {\n" + + " \treturn;\n" + + " \n" + + " \tx2();\n" + + "-}\n" + + "-\n" + + "-func() {\n" + + "-\n" + + "- \n" + + "- \n" + + "- \n" + + " }\n" + + " \n" + + " func() {\n" + testing.expectf(t, s == expected, "got:\n%s\nexpected:\n%s", s, expected) + + // Without the heuristic the deletions sit lower down + plain := must(diff.diff_text(a, b, context.allocator, context.allocator, opts = {flags = {.Prune, .Compact}})) + defer diff.text_diff_destroy(plain, context.allocator) + testing.expect(t, must(diff.unified(plain, context.temp_allocator)) != expected, "the heuristic made no difference, meaning this test shows nothing") +} + +// Scratch of exactly `scratch_size_text` bytes, counting the indents, must work at every alignment. +@(test) +test_indent_scratch_alignment :: proc(t: ^testing.T) { + rand.reset(t.seed) + a, b := code_pair() + opts := diff.Options{flags = {.Prune, .Compact, .Indent_Heuristic}} + n, m := diff.token_count(a, .Line), diff.token_count(b, .Line) + size := diff.scratch_size_text(n, m, opts) + buf := make([]byte, size + 16) + defer delete(buf) + + expected := must(diff.diff_text(a, b, context.allocator, context.allocator, opts = opts)) + defer diff.text_diff_destroy(expected, context.allocator) + for offset in 0..<16 { + c := must(diff.changes_make(n, m, context.allocator)) + defer diff.changes_destroy(c, context.allocator) + d := diff.diff_text_buffer(a, b, c, buf[offset:][:size], opts = opts) + testing.expectf(t, slice.equal(d.changes.bits, expected.changes.bits), "offset %d differs", offset) + } +} + diff --git a/tests/core/diff/test_core_diff_prune.odin b/tests/core/diff/test_core_diff_prune.odin new file mode 100644 index 000000000..e84cc5c9b --- /dev/null +++ b/tests/core/diff/test_core_diff_prune.odin @@ -0,0 +1,208 @@ +package test_core_diff + +import "core:diff" +import "core:math/rand" +import "core:slice" +import "core:testing" + +/* + Harness: the documented steps of `.Prune`, for inputs where its rule for frequent tokens sets nothing aside. + The git fixtures check that rule. +*/ + +/* + `.Prune` must give what its steps give, done the long way: set aside the common prefix and suffix, then every + token absent from the other side, and diff what is left with the same algorithm. +*/ +check_prune_steps :: proc(t: ^testing.T, a, b: []i32, algorithm: diff.Algorithm, loc := #caller_location) -> bool { + n, m := len(a), len(b) + + prefix := 0 + for prefix < min(n, m) && a[prefix] == b[prefix] { + prefix += 1 + } + suffix := 0 + for suffix < min(n, m) - prefix && a[n-1-suffix] == b[m-1-suffix] { + suffix += 1 + } + + deleted := make([]bool, n, context.temp_allocator) + inserted := make([]bool, m, context.temp_allocator) + kept_a := make([dynamic]i32, context.temp_allocator) + kept_b := make([dynamic]i32, context.temp_allocator) + index_a := make([dynamic]int, context.temp_allocator) + index_b := make([dynamic]int, context.temp_allocator) + for i in prefix.. []i32 { + s := make([]i32, length) + next_unique := unique_from + for &x in s { + if rand.int_max(unique_chance) == 0 { + x = next_unique + next_unique += 1 + } else { + x = i32(rand.int_max(frequent)) + } + } + return s +} + + +/* + Tests +*/ + +/* + With no frequent tokens, `.Prune` only sets aside the common ends and the tokens with no match. Here each token + occurs at most 3 times, and each side has at least 16. The limit for a frequent token is then 8 or more. +*/ +@(test) +test_prune_without_frequent :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<200 { + // Each of `length/2` tokens 3 times, shuffled, and the first `length` of them + length := 24 + rand.int_max(200) + pool := make([]i32, 3*(length/2)) + defer delete(pool) + for &x, k in pool { + x = i32(k / 3) + } + rand.shuffle(pool) + a := pool[:length] + + // Tokens removed, moved, or replaced by new ones found nowhere else + b := make([dynamic]i32, 0, length) + defer delete(b) + append(&b, ..a) + next := i32(10_000) + for _ in 0.. (i, j, k: int) { + for x in a_lo.. k { + i, j, k = x, y, length + } + } + } + return +} + +// Ratcliff/Obershelp the plain recursive way, with no budget. +reference_ratcliff_obershelp :: proc(a, b: []i32, deleted, inserted: []bool, a_lo, a_hi, b_lo, b_hi: int) { + i, j, k := reference_longest_block(a, b, a_lo, a_hi, b_lo, b_hi) + if k == 0 { + for x in a_lo.. bool { + deleted := make([]bool, len(a), context.temp_allocator) + inserted := make([]bool, len(b), context.temp_allocator) + reference_ratcliff_obershelp(a, b, deleted, inserted, 0, len(a), 0, len(b)) + + c := diff_exact(a, b, {algorithm = .Ratcliff_Obershelp, max_cost = 1 << 30}) + defer diff.changes_destroy(c, context.allocator) + for x in 0.. bool { + _, _, longest := reference_longest_block(a, b, 0, len(a), 0, len(b)) + + c := diff_exact(a, b, {algorithm = .Ratcliff_Obershelp, max_cost = 1 << 30}) + defer diff.changes_destroy(c, context.allocator) + + run := 0 + it := diff.edit_iterator_make(c) + for e in diff.edit_iterate(&it) { + if e.op == .Equal { + run = max(run, e.a_hi - e.a_lo) + } + } + return testing.expectf(t, run == longest, "longest equal run %d, longest common block %d (a = %v, b = %v)", run, longest, a, b, loc = loc) +} + + +/* + Tests +*/ + +@(test) +test_ratcliff_obershelp_matches_reference :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<300 { + alphabet := 1 + rand.int_max(8) + a := random_sequence(rand.int_max(60), alphabet) + defer delete(a) + b := mutate(a, rand.int_max(20), alphabet) + defer delete(b) + if !check_ratcliff_obershelp_matches_reference(t, a, b) || !check_longest_run(t, a, b) { + return + } + } + for _ in 0..<100 { + alphabet := 1 + rand.int_max(4) + a := random_sequence(rand.int_max(40), alphabet) + defer delete(a) + b := random_sequence(rand.int_max(40), alphabet) + defer delete(b) + if !check_ratcliff_obershelp_matches_reference(t, a, b) || !check_longest_run(t, a, b) { + return + } + } +} + +@(test) +test_ratcliff_obershelp_exhaustive :: proc(t: ^testing.T) { + check_exhaustive(t, 2, 7, {algorithm = .Ratcliff_Obershelp}) + check_exhaustive(t, 3, 5, {algorithm = .Ratcliff_Obershelp}) +} + +// The longest block wins over more matches spread out: "XYZ" against six single tokens. +@(test) +test_ratcliff_obershelp_prefers_blocks :: proc(t: ^testing.T) { + ONE, TWO, THREE, FOUR, FIVE, SIX, X, Y, Z :: 1, 2, 3, 4, 5, 6, 24, 25, 26 + A, B, C, D, E, F :: 11, 12, 13, 14, 15, 16 + a := []i32{ONE, TWO, THREE, FOUR, FIVE, SIX, X, Y, Z} + b := []i32{X, Y, Z, ONE, A, TWO, B, THREE, C, FOUR, D, FIVE, E, SIX, F} + + ro := diff_exact(a, b, {algorithm = .Ratcliff_Obershelp}) + defer diff.changes_destroy(ro, context.allocator) + myers := diff_exact(a, b, {algorithm = .Myers}) + defer diff.changes_destroy(myers, context.allocator) + + testing.expect_value(t, diff.stats(ro).equal, 3) + testing.expect_value(t, diff.stats(myers).equal, 6) +} + +/* + Inputs ending in 20 of a token found nowhere else: searching those is 400 steps, far past a budget of + `1 * (n+m)`, meaning every diff goes to Myers with the same cap. At least one must come out differently + without the budget, or the test shows nothing. +*/ +@(test) +test_ratcliff_obershelp_budget :: proc(t: ^testing.T) { + REPEATED :: 99 + + rand.reset(t.seed) + differs := false + for _ in 0..<50 { + a := make([dynamic]i32) + defer delete(a) + b := make([dynamic]i32) + defer delete(b) + + base := random_sequence(30, 6) + defer delete(base) + changed := mutate(base, 8, 6) + defer delete(changed) + append(&a, ..base) + append(&b, ..changed) + for _ in 0..<20 { + append(&a, REPEATED) + append(&b, REPEATED) + } + + budgeted := diff_exact(a[:], b[:], {algorithm = .Ratcliff_Obershelp, max_cost = 1}) + defer diff.changes_destroy(budgeted, context.allocator) + unlimited := diff_exact(a[:], b[:], {algorithm = .Ratcliff_Obershelp, max_cost = 1 << 30}) + defer diff.changes_destroy(unlimited, context.allocator) + myers := diff_exact(a[:], b[:], {algorithm = .Myers, max_cost = 1}) + defer diff.changes_destroy(myers, context.allocator) + + // A cost too large to multiply by the tokens is no limit either, rather than an overflow + maxed := diff_exact(a[:], b[:], {algorithm = .Ratcliff_Obershelp, max_cost = max(int)}) + defer diff.changes_destroy(maxed, context.allocator) + + if !check_valid(t, a[:], b[:], budgeted) || !testing.expect(t, slice.equal(budgeted.bits, myers.bits), "running out of budget did not hand the diff to Myers") { + return + } + if !testing.expect(t, slice.equal(maxed.bits, unlimited.bits), "a cost of max(int) did not mean no limit") { + return + } + differs ||= !slice.equal(unlimited.bits, myers.bits) + } + testing.expect(t, differs, "without a budget every result was the same, meaning this test shows nothing") +} + + +/* + Frequent tokens: one occurring in `b` more than 64 times may carry a run on but never start one. + The longest such run, ending earliest, is grown back through the equal tokens before it. + When only frequent tokens match, the region goes to Myers. +*/ + +reference_frequent :: proc(b: []i32) -> map[i32]bool { + count := make(map[i32]int, context.temp_allocator) + for y in b { + count[y] += 1 + } + frequent := make(map[i32]bool, context.temp_allocator) + for id, c in count { + if c > 64 { + frequent[id] = true + } + } + return frequent +} + +reference_ratcliff_obershelp_frequent :: proc(a, b: []i32, deleted, inserted: []bool, a_lo, a_hi, b_lo, b_hi: int, frequent: map[i32]bool, id_count: int) { + if a_lo == a_hi || b_lo == b_hi { + for x in a_lo.. int { return (x - a_lo + 1)*w + (y - b_lo + 1) } + frequent_in_common := false + i, j, k := 0, 0, 0 + for x in a_lo.. k { + i, j, k = x - r + 1, y - r + 1, r + } + } + } + + if k == 0 { + if frequent_in_common { + c := must(diff.diff_interned(a[a_lo:a_hi], b[b_lo:b_hi], id_count, context.allocator, context.allocator, opts = {algorithm = .Myers, max_cost = 1 << 30})) + defer diff.changes_destroy(c, context.allocator) + for x in a_lo.. a_lo && j > b_lo && a[i-1] == b[j-1] { + i -= 1 + j -= 1 + k += 1 + } + reference_ratcliff_obershelp_frequent(a, b, deleted, inserted, a_lo, i, b_lo, j, frequent, id_count) + reference_ratcliff_obershelp_frequent(a, b, deleted, inserted, i + k, a_hi, j + k, b_hi, frequent, id_count) +} + +// Three tokens making up a quarter each, far past 64 occurrences, and many more that are rare. +frequent_and_rare :: proc(length: int) -> []i32 { + s := make([]i32, length) + for &x in s { + if rand.int_max(4) < 3 { + x = i32(rand.int_max(3)) + } else { + x = i32(100 + rand.int_max(300)) + } + } + return s +} + +@(test) +test_ratcliff_obershelp_frequent_matches_reference :: proc(t: ^testing.T) { + rand.reset(t.seed) + checked := 0 + for _ in 0..<40 { + a := frequent_and_rare(260 + rand.int_max(100)) + defer delete(a) + b := mutate(a, rand.int_max(40), 3) + defer delete(b) + + frequent := reference_frequent(b) + deleted := make([]bool, len(a), context.temp_allocator) + inserted := make([]bool, len(b), context.temp_allocator) + reference_ratcliff_obershelp_frequent(a, b, deleted, inserted, 0, len(a), 0, len(b), frequent, id_count_of(a, b)) + + c := diff_exact(a, b, {algorithm = .Ratcliff_Obershelp, max_cost = 1 << 30}) + defer diff.changes_destroy(c, context.allocator) + if !check_valid(t, a, b, c) { + return + } + for x in 0.. 0 { + checked += 1 + } + } + testing.expect(t, checked > 0, "no input had frequent tokens, meaning this test shows nothing") +} + +// Only frequent tokens in common: the region goes to Myers rather than being all changes. +@(test) +test_ratcliff_obershelp_only_frequent :: proc(t: ^testing.T) { + a := make([]i32, 300) + defer delete(a) + b := make([]i32, 300) + defer delete(b) + for i in 0..<300 { + a[i] = i32(1000 + i) // all different, and found nowhere in `b` + b[i] = i32(2000 + i) + } + for i := 0; i < 300; i += 3 { + a[i] = 7 // 100 times in `b`, and the only token in common + b[i] = 7 + } + + c := diff_exact(a, b, {algorithm = .Ratcliff_Obershelp}) + defer diff.changes_destroy(c, context.allocator) + check_valid(t, a, b, c) + testing.expect_value(t, diff.stats(c).equal, 100) +} + +// A rare token in the wrong place must not win over a long run that is mostly frequent tokens. +@(test) +test_ratcliff_obershelp_frequent_runs_count :: proc(t: ^testing.T) { + RARE :: 500 + a := make([dynamic]i32) + defer delete(a) + b := make([dynamic]i32) + defer delete(b) + + // `a` opens with a rare token that `b` only has at its very end, then both share one long stretch + // of frequent tokens started by another rare token. Counting runs of one rare token alone, the first + // would win and leave nothing else to match. + append(&a, RARE, RARE + 1) + append(&b, RARE + 1) + for k in 0..<300 { + append(&a, i32(k % 3)) + append(&b, i32(k % 3)) + } + append(&b, RARE) + + c := diff_exact(a[:], b[:], {algorithm = .Ratcliff_Obershelp}) + defer diff.changes_destroy(c, context.allocator) + check_valid(t, a[:], b[:], c) + testing.expect_value(t, diff.stats(c).equal, 301) +} diff --git a/tests/core/diff/test_core_diff_text.odin b/tests/core/diff/test_core_diff_text.odin new file mode 100644 index 000000000..31c2cb668 --- /dev/null +++ b/tests/core/diff/test_core_diff_text.odin @@ -0,0 +1,447 @@ +package test_core_diff + +import "core:diff" +import "core:math/rand" +import "core:slice" +import "core:strings" +import "core:testing" +import "core:unicode" +import "core:unicode/utf8" + +/* + Harness +*/ + +// Lines split with `core:strings`, independently of `core:diff`'s tokeniser. +split_lines :: proc(s: string, allocator := context.temp_allocator) -> []string { + lines := strings.split_after(s, "\n", allocator) + if len(lines) > 0 && lines[len(lines) - 1] == "" { + return lines[:len(lines) - 1] + } + return lines +} + +/* + Applies `flags` to one line the long way: an independent version of what `diff_text` must treat as equal. + Case folding uses `strings.to_lower`, which agrees with simple folding on ASCII. +*/ +normalize_line :: proc(line: string, flags: diff.Text_Flags, allocator := context.temp_allocator) -> string { + body, newline := line, "" + if strings.has_suffix(body, "\n") { + body, newline = body[:len(body) - 1], "\n" + if .Ignore_CR in flags && strings.has_suffix(body, "\r") { + body = body[:len(body) - 1] + } + } + if flags & {.Ignore_Trailing_Space, .Ignore_All_Space} != {} { + body = strings.trim_right_proc(body, unicode.is_space) + } + if .Ignore_All_Space in flags { + b := strings.builder_make(allocator) + for r in body { + if !unicode.is_space(r) { + strings.write_rune(&b, r) + } + } + body = strings.to_string(b) + } + if .Ignore_Case in flags { + body = strings.to_lower(body, allocator) + } + return strings.concatenate({body, newline}, allocator) +} + +// `diff_text` must give exactly the bits of a generic diff of the lines, normalised the long way. +check_text_matches_lines :: proc(t: ^testing.T, a, b: string, flags: diff.Text_Flags, opts := diff.Options{}, loc := #caller_location) -> bool { + d := must(diff.diff_text(a, b, context.allocator, context.allocator, flags = flags, opts = opts)) + defer diff.text_diff_destroy(d, context.allocator) + + lines_a := split_lines(a) + lines_b := split_lines(b) + for &line in lines_a { + line = normalize_line(line, flags) + } + for &line in lines_b { + line = normalize_line(line, flags) + } + + expected := must(diff.diff(lines_a, lines_b, context.allocator, context.allocator, opts = opts)) + defer diff.changes_destroy(expected, context.allocator) + testing.expectf(t, slice.equal(d.changes.bits, expected.bits), "%v: diff_text differs from diffing the lines (a = %q, b = %q)", flags, a, b, loc = loc) or_return + return check_spans(t, d, flags == {}, loc) +} + +/* + Spans cover the text in whole tokens: the `.Equal` and `.Delete` spans rebuild `a`, and when nothing is ignored + the `.Equal` and `.Insert` spans rebuild `b`. +*/ +check_spans :: proc(t: ^testing.T, d: diff.Text_Diff, exact: bool, loc := #caller_location) -> bool { + rebuilt_a := strings.builder_make(context.temp_allocator) + rebuilt_b := strings.builder_make(context.temp_allocator) + spans := 0 + + it := diff.span_iterator_make(d) + for s in diff.span_iterate(&it) { + testing.expectf(t, len(s.text) > 0, "span %d is empty", spans, loc = loc) or_return + switch s.op { + case .Equal: + strings.write_string(&rebuilt_a, s.text) + strings.write_string(&rebuilt_b, s.text) + case .Delete: + strings.write_string(&rebuilt_a, s.text) + case .Insert: + strings.write_string(&rebuilt_b, s.text) + } + spans += 1 + } + testing.expect_value(t, spans, diff.edit_count(d.changes), loc = loc) or_return + testing.expectf(t, strings.to_string(rebuilt_a) == d.a, "spans rebuild a as %q, not %q", strings.to_string(rebuilt_a), d.a, loc = loc) or_return + if exact { + testing.expectf(t, strings.to_string(rebuilt_b) == d.b, "spans rebuild b as %q, not %q", strings.to_string(rebuilt_b), d.b, loc = loc) or_return + } + return true +} + +collect_spans :: proc(d: diff.Text_Diff) -> [dynamic]diff.Span { + spans: [dynamic]diff.Span + it := diff.span_iterator_make(d) + for s in diff.span_iterate(&it) { + append(&spans, s) + } + return spans +} + +expect_text_stats :: proc(t: ^testing.T, a, b: string, token: diff.Token, flags: diff.Text_Flags, expected: diff.Stats, loc := #caller_location) { + d := must(diff.diff_text(a, b, context.allocator, context.allocator, token = token, flags = flags)) + defer diff.text_diff_destroy(d, context.allocator) + testing.expectf(t, diff.stats(d.changes) == expected, "%v %v: %q against %q gave %v, expected %v", token, flags, a, b, diff.stats(d.changes), expected, loc = loc) +} + +/* + The tokens of `s` split the long way, decoding rune by rune and classing each with `core:unicode`, + independently of the tokeniser's ASCII fast paths. Invalid UTF-8 decodes as one rune per byte. +*/ +reference_tokens :: proc(s: string, token: diff.Token, allocator := context.temp_allocator) -> []string { + Class :: enum {Other, Word, Space} + class_of :: proc(r: rune) -> Class { + switch { + case r == '_' || unicode.is_letter(r) || unicode.is_digit(r): + return .Word + case unicode.is_space(r): + return .Space + } + return .Other + } + + tokens := make([dynamic]string, allocator) + start := 0 + prev := Class.Other + for pos := 0; pos < len(s); /**/ { + r, width := utf8.decode_rune_in_string(s[pos:]) + class := class_of(r) + if pos > start && (token == .Rune || class == .Other || class != prev) { + append(&tokens, s[start:pos]) + start = pos + } + prev = class + pos += width + } + if start < len(s) { + append(&tokens, s[start:]) + } + return tokens[:] +} + +// Interns the tokens of both sides, for comparing against `lcs_length`. +intern_tokens :: proc(a, b: []string, allocator := context.temp_allocator) -> (x, y: []i32) { + ids := make(map[string]i32, allocator) + x = make([]i32, len(a), allocator) + y = make([]i32, len(b), allocator) + for token, k in a { + x[k] = ids[token] or_else i32(len(ids)) + ids[token] = x[k] + } + for token, k in b { + y[k] = ids[token] or_else i32(len(ids)) + ids[token] = y[k] + } + return +} + +// ASCII and not, words, spaces and punctuation, and invalid UTF-8, including a sequence cut short. +@(rodata) +TOKEN_PIECES := [?]string{ + "a", "Z", "_", "7", "foo", ",", "(", " ", " ", "\t", "\r", "\n", + "é", "ж", "日", "€", "😀", "\u00a0", "\u2003", "\u0085", "\xff", "\x80", "\xe2\x82", +} + +random_pieces :: proc(count: int, allocator := context.temp_allocator) -> string { + b := strings.builder_make(allocator) + for _ in 0.. string { + b := strings.builder_make(allocator) + for k in 0.. string { + lines := make([dynamic]string, context.temp_allocator) + append(&lines, ..split_lines(s)) + for _ in 0.. 0 { + ordered_remove(&lines, rand.int_max(len(lines))) + } + case 2: + if len(lines) > 0 { + lines[rand.int_max(len(lines))] = random_text(1, allocator) + } + } + } + // Every line but the last needs its "\n" back, wherever it moved to + b := strings.builder_make(allocator) + for line, k in lines { + strings.write_string(&b, line) + if k < len(lines) - 1 && !strings.has_suffix(line, "\n") { + strings.write_byte(&b, '\n') + } + } + return strings.to_string(b) +} + +@(rodata) +ALL_FLAG_SETS := [?]diff.Text_Flags{ + {}, + {.Ignore_CR}, + {.Ignore_Trailing_Space}, + {.Ignore_All_Space}, + {.Ignore_Case}, + {.Ignore_CR, .Ignore_Case}, + {.Ignore_Trailing_Space, .Ignore_Case}, + {.Ignore_CR, .Ignore_Trailing_Space, .Ignore_All_Space, .Ignore_Case}, +} + + +/* + Tests +*/ + +@(test) +test_token_count :: proc(t: ^testing.T) { + Case :: struct { + s: string, + lines, words, runes: int, + } + cases := []Case{ + {"", 0, 0, 0}, + {"\n", 1, 1, 1}, + {"a", 1, 1, 1}, + {"a\n", 1, 2, 2}, + {"a\r\n", 1, 2, 3}, + {"a\n\n", 2, 2, 3}, + {"a\nb", 2, 3, 3}, + {"hello, world", 1, 4, 12}, + {"foo_bar 123 baz", 1, 5, 16}, + {"héllo wörld", 1, 3, 11}, + {"aé€😀", 1, 3, 4}, + {"x\xff\xfey", 1, 4, 4}, + {"tab\tnew\r\nline\n", 2, 6, 14}, + } + for c in cases { + testing.expectf(t, diff.token_count(c.s, .Line) == c.lines, "%q: %d lines, expected %d", c.s, diff.token_count(c.s, .Line), c.lines) + testing.expectf(t, diff.token_count(c.s, .Word) == c.words, "%q: %d words, expected %d", c.s, diff.token_count(c.s, .Word), c.words) + testing.expectf(t, diff.token_count(c.s, .Rune) == c.runes, "%q: %d runes, expected %d", c.s, diff.token_count(c.s, .Rune), c.runes) + } +} + +// Words and runes must split as decoding rune by rune does, both to count them and to diff them. +@(test) +test_tokens_random :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<300 { + a := random_pieces(rand.int_max(40)) + b := random_pieces(rand.int_max(40)) + for token in ([]diff.Token{.Word, .Rune}) { + tokens_a := reference_tokens(a, token) + tokens_b := reference_tokens(b, token) + if !testing.expectf(t, diff.token_count(a, token) == len(tokens_a), "%v: %q counts %d, expected %d", token, a, diff.token_count(a, token), len(tokens_a)) { + return + } + + // Being minimal, the diff only comes out as small as expected with the same tokens + d := must(diff.diff_text(a, b, context.allocator, context.allocator, token = token, opts = {algorithm = .Minimal})) + defer diff.text_diff_destroy(d, context.allocator) + x, y := intern_tokens(tokens_a, tokens_b) + got, expected := cost(d.changes), len(x) + len(y) - 2*lcs_length(x, y) + if !testing.expectf(t, got == expected, "%v: %q against %q changes %d tokens, expected %d", token, a, b, got, expected) || !check_spans(t, d, true) { + return + } + } + } +} + +@(test) +test_text_lines :: proc(t: ^testing.T) { + cases := [][2]string{ + {"", ""}, + {"", "a\n"}, + {"a\n", ""}, + {"a\nb\nc\n", "a\nb\nc\n"}, + {"a\nb\nc\n", "a\nB\nc\n"}, + {"a\nb\nc", "a\nb\nc\n"}, + {"a\r\nb\r\n", "a\nb\n"}, + {"one\ntwo\nthree\n", "zero\none\nthree\nfour"}, + } + for c in cases { + check_text_matches_lines(t, c[0], c[1], {}) + } +} + +@(test) +test_text_random :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<150 { + a := random_text(rand.int_max(60)) + b := mutate_lines(a, rand.int_max(15)) + for flags in ALL_FLAG_SETS { + if !check_text_matches_lines(t, a, b, flags) { + return + } + } + } +} + +@(test) +test_text_flags :: proc(t: ^testing.T) { + // A final "\n" is never ignored + expect_text_stats(t, "a\n", "a", .Line, {.Ignore_All_Space}, {deleted = 1, inserted = 1}) + expect_text_stats(t, "a ", "a\n", .Line, {.Ignore_Trailing_Space}, {deleted = 1, inserted = 1}) + expect_text_stats(t, "a ", "a", .Line, {.Ignore_Trailing_Space}, {equal = 1}) + + // Only a "\r" right before the "\n" + expect_text_stats(t, "a\r\nb\r\n", "a\nb\n", .Line, {.Ignore_CR}, {equal = 2}) + expect_text_stats(t, "a\r\nb\r\n", "a\nb\n", .Line, {}, {deleted = 2, inserted = 2}) + expect_text_stats(t, "a\rb\n", "ab\n", .Line, {.Ignore_CR}, {deleted = 1, inserted = 1}) + + // Trailing against all whitespace + expect_text_stats(t, "a b \t\n", "a b\n", .Line, {.Ignore_Trailing_Space}, {equal = 1}) + expect_text_stats(t, "a b\n", "ab\n", .Line, {.Ignore_Trailing_Space}, {deleted = 1, inserted = 1}) + expect_text_stats(t, "a b\n", "ab\n", .Line, {.Ignore_All_Space}, {equal = 1}) + expect_text_stats(t, "\t\n", "\n", .Line, {.Ignore_All_Space}, {equal = 1}) + + // Simple case folding, beyond ASCII + expect_text_stats(t, "Hello\n", "hELLO\n", .Line, {.Ignore_Case}, {equal = 1}) + expect_text_stats(t, "K\n", "k\n", .Line, {.Ignore_Case}, {equal = 1}) // Kelvin sign + expect_text_stats(t, "ſ\n", "S\n", .Line, {.Ignore_Case}, {equal = 1}) // long s + expect_text_stats(t, "ΣΑΣ\n", "σας\n", .Line, {.Ignore_Case}, {equal = 1}) // final sigma is a simple fold of Σ + expect_text_stats(t, "ß\n", "ss\n", .Line, {.Ignore_Case}, {deleted = 1, inserted = 1}) // but "ss" is only a full fold of ß + + // Flags apply to words and runes too + expect_text_stats(t, "The quick", "the quick", .Word, {.Ignore_Case, .Ignore_All_Space}, {equal = 3}) + expect_text_stats(t, "aBc", "AbC", .Rune, {.Ignore_Case}, {equal = 3}) +} + +@(test) +test_text_words :: proc(t: ^testing.T) { + d := must(diff.diff_text("the quick brown fox", "the slow brown fox", context.allocator, context.allocator, token = .Word)) + defer diff.text_diff_destroy(d, context.allocator) + spans := collect_spans(d) + defer delete(spans) + + expected := []diff.Span{ + {.Equal, "the "}, + {.Delete, "quick"}, + {.Insert, "slow"}, + {.Equal, " brown fox"}, + } + testing.expectf(t, slice.equal(spans[:], expected), "spans %v, expected %v", spans, expected) + check_spans(t, d, true) +} + +@(test) +test_text_runes :: proc(t: ^testing.T) { + d := must(diff.diff_text("naïve 😀 x\xff", "naive 😀 y\xff", context.allocator, context.allocator, token = .Rune)) + defer diff.text_diff_destroy(d, context.allocator) + spans := collect_spans(d) + defer delete(spans) + + expected := []diff.Span{ + {.Equal, "na"}, + {.Delete, "ï"}, + {.Insert, "i"}, + {.Equal, "ve 😀 "}, + {.Delete, "x"}, + {.Insert, "y"}, + {.Equal, "\xff"}, + } + testing.expectf(t, slice.equal(spans[:], expected), "spans %v, expected %v", spans, expected) + check_spans(t, d, true) +} + +@(test) +test_text_large :: proc(t: ^testing.T) { + rand.reset(t.seed) + a := random_text(5_000) + b := mutate_lines(a, 200) + check_text_matches_lines(t, a, b, {}) + check_text_matches_lines(t, a, b, {.Ignore_All_Space, .Ignore_Case}) + check_text_matches_lines(t, a, b, {}, {algorithm = .Minimal}) +} + +// Scratch of exactly `scratch_size_text` bytes must work at every alignment, for every kind of token. +@(test) +test_text_scratch_alignment :: proc(t: ^testing.T) { + rand.reset(t.seed) + a := random_text(200) + b := mutate_lines(a, 30) + + for token in diff.Token { + n, m := diff.token_count(a, token), diff.token_count(b, token) + size := diff.scratch_size_text(n, m) + buf := make([]byte, size + 16) + defer delete(buf) + + expected := must(diff.diff_text(a, b, context.allocator, context.allocator, token = token)) + defer diff.text_diff_destroy(expected, context.allocator) + for offset in 0..<16 { + c := must(diff.changes_make(n, m, context.allocator)) + defer diff.changes_destroy(c, context.allocator) + d := diff.diff_text_buffer(a, b, c, buf[offset:][:size], token) + testing.expectf(t, slice.equal(d.changes.bits, expected.changes.bits), "%v at offset %d differs", token, offset) + } + } +} + +@(test) +test_text_allocations :: proc(t: ^testing.T) { + result := Counting_Allocator{backing = context.allocator} + transient := Counting_Allocator{backing = context.allocator} + + d := must(diff.diff_text("a\nb\nc\n", "a\nc\nd\n", counting_allocator(&result), counting_allocator(&transient))) + testing.expect_value(t, result.allocs, 1) + testing.expect_value(t, transient.allocs, 1) + testing.expect_value(t, transient.frees, 1) + testing.expect_value(t, diff.stats(d.changes), diff.Stats{equal = 2, deleted = 1, inserted = 1}) + diff.text_diff_destroy(d, counting_allocator(&result)) + testing.expect_value(t, result.frees, 1) +} diff --git a/tests/core/diff/test_core_diff_unified.odin b/tests/core/diff/test_core_diff_unified.odin new file mode 100644 index 000000000..264179721 --- /dev/null +++ b/tests/core/diff/test_core_diff_unified.odin @@ -0,0 +1,459 @@ +package test_core_diff + +import "core:diff" +import "core:io" +import "core:math/rand" +import "core:mem" +import "core:strconv" +import "core:strings" +import "core:testing" + +/* + Harness +*/ + +// Parses "start[,count]". +parse_range :: proc(s: string) -> (lo, count: int, ok: bool) { + start_text, _, count_text := strings.partition(s, ",") + start := strconv.parse_int(start_text, 10) or_return + count = 1 + if count_text != "" { + count = strconv.parse_int(count_text, 10) or_return + } + // An empty range starts at the line before it + lo = count == 0 ? start : start - 1 + return lo, count, true +} + +/* + Applies a unified diff to `a` the simple way, checking as it goes that every context and deleted line + matches `a`, that the hunk headers agree with the lines under them, and that hunks are in order. +*/ +apply_unified :: proc(t: ^testing.T, a, patch: string, loc := #caller_location) -> (b: string, ok: bool) { + if patch == "" { + return a, true + } + lines_a := split_lines(a) + p := split_lines(patch) + testing.expectf(t, len(p) >= 2 && strings.has_prefix(p[0], "--- ") && strings.has_prefix(p[1], "+++ "), "bad file headers in %q", patch, loc = loc) or_return + + out := strings.builder_make(context.temp_allocator) + out_lines := 0 + ai := 0 + k := 2 + for k < len(p) { + header := strings.trim_suffix(p[k], "\n") + testing.expectf(t, strings.has_prefix(header, "@@ -") && strings.has_suffix(header, " @@"), "bad hunk header %q", header, loc = loc) or_return + ranges := header[len("@@ -"):len(header) - len(" @@")] + a_text, _, b_text := strings.partition(ranges, " +") + a_lo, a_count, a_ok := parse_range(a_text) + b_lo, b_count, b_ok := parse_range(b_text) + testing.expectf(t, a_ok && b_ok, "bad ranges in %q", header, loc = loc) or_return + testing.expectf(t, a_lo >= ai, "hunk %q goes back to line %d", header, ai, loc = loc) or_return + + for ai < a_lo { + strings.write_string(&out, lines_a[ai]) + ai += 1 + out_lines += 1 + } + testing.expectf(t, out_lines == b_lo, "hunk %q starts at b line %d, not %d", header, out_lines, b_lo, loc = loc) or_return + + k += 1 + seen_a, seen_b := 0, 0 + for k < len(p) && !strings.has_prefix(p[k], "@@") { + line := p[k] + body := line[1:] + if k + 1 < len(p) && strings.has_prefix(p[k + 1], "\\ No newline at end of file") { + body = strings.trim_suffix(body, "\n") + k += 1 + } + switch line[0] { + case ' ', '-': + testing.expectf(t, ai < len(lines_a) && lines_a[ai] == body, "line %q does not match a[%d]", line, ai, loc = loc) or_return + if line[0] == ' ' { + strings.write_string(&out, body) + out_lines += 1 + seen_b += 1 + } + ai += 1 + seen_a += 1 + case '+': + strings.write_string(&out, body) + out_lines += 1 + seen_b += 1 + case: + testing.expectf(t, false, "bad patch line %q", line, loc = loc) or_return + } + k += 1 + } + testing.expectf(t, seen_a == a_count && seen_b == b_count, "hunk %q holds %d and %d lines", header, seen_a, seen_b, loc = loc) or_return + } + for ai < len(lines_a) { + strings.write_string(&out, lines_a[ai]) + ai += 1 + } + return strings.to_string(out), true +} + +// The line numbers and text the iterator gives must match the lines of `a` and `b`. +check_unified_lines :: proc(t: ^testing.T, d: diff.Text_Diff, context_len: int, loc := #caller_location) -> bool { + lines_a := split_lines(d.a) + lines_b := split_lines(d.b) + it := diff.unified_iterator_make(d, context_len) + for line in diff.unified_iterate(&it) { + if line.a_line > 0 { + testing.expectf(t, strings.trim_suffix(lines_a[line.a_line - 1], "\n") == line.text, "%v is not a line %d", line, line.a_line, loc = loc) or_return + } + if line.b_line > 0 && line.kind == .Insert { + testing.expectf(t, strings.trim_suffix(lines_b[line.b_line - 1], "\n") == line.text, "%v is not b line %d", line, line.b_line, loc = loc) or_return + } + switch line.kind { + case .Equal: + testing.expectf(t, line.a_line > 0 && line.b_line > 0, "%v lacks a line number", line, loc = loc) or_return + case .Delete: + testing.expectf(t, line.a_line > 0 && line.b_line == 0, "%v has the wrong line numbers", line, loc = loc) or_return + case .Insert: + testing.expectf(t, line.a_line == 0 && line.b_line > 0, "%v has the wrong line numbers", line, loc = loc) or_return + case .Hunk_Header, .No_Newline: + testing.expectf(t, line.a_line == 0 && line.b_line == 0 && line.text == "", "%v should carry no line", line, loc = loc) or_return + } + } + return true +} + +// Every way of producing the output must agree, and applying it to `a` must give `b`. +check_unified :: proc(t: ^testing.T, a, b: string, context_len: int, flags := diff.Text_Flags{}, loc := #caller_location) -> bool { + d := must(diff.diff_text(a, b, context.allocator, context.allocator, flags = flags)) + defer diff.text_diff_destroy(d, context.allocator) + + s := must(diff.unified(d, context.allocator, context_len = context_len)) + defer delete(s) + testing.expect_value(t, diff.unified_size(d, context_len = context_len), len(s), loc = loc) or_return + + sb := strings.builder_make() + defer strings.builder_destroy(&sb) + testing.expect_value(t, diff.write_unified_to_builder(&sb, d, context_len = context_len), nil, loc = loc) or_return + testing.expect_value(t, strings.to_string(sb), s, loc = loc) or_return + testing.expect_value(t, cap(sb.buf), len(s), loc = loc) or_return + + // Growing as it writes gives the same, fitted to it + grown := must(diff.unified(d, context.allocator, context_len = context_len, sizing = .Grow)) + defer delete(grown) + testing.expect_value(t, grown, s, loc = loc) or_return + + grown_sb := strings.builder_make() + defer strings.builder_destroy(&grown_sb) + testing.expect_value(t, diff.write_unified_to_builder(&grown_sb, d, context_len = context_len, sizing = .Grow), nil, loc = loc) or_return + testing.expect_value(t, strings.to_string(grown_sb), s, loc = loc) or_return + + check_unified_lines(t, d, context_len, loc) or_return + + rebuilt := apply_unified(t, a, s, loc) or_return + if flags == {} { + testing.expectf(t, rebuilt == b, "patch rebuilt %q, not %q, from:\n%s", rebuilt, b, s, loc = loc) or_return + } else { + testing.expectf(t, len(split_lines(rebuilt)) == len(split_lines(b)), "patch rebuilt %q, which does not line up with %q", rebuilt, b, loc = loc) or_return + } + return true +} + +expect_unified :: proc(t: ^testing.T, a, b, expected: string, context_len := diff.DEFAULT_CONTEXT_LEN, a_name := "a", b_name := "b", style := diff.STYLE_NONE, loc := #caller_location) { + d := must(diff.diff_text(a, b, context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + s := must(diff.unified(d, context.allocator, a_name, b_name, context_len, style)) + defer delete(s) + testing.expectf(t, s == expected, "got:\n%s\nexpected:\n%s", s, expected, loc = loc) +} + + +/* + Tests +*/ + +@(test) +test_unified_basic :: proc(t: ^testing.T) { + expect_unified(t, "a\nb\nc\n", "a\nB\nc\n", + "--- a\n" + + "+++ b\n" + + "@@ -1,3 +1,3 @@\n" + + " a\n" + + "-b\n" + + "+B\n" + + " c\n") + + expect_unified(t, "a\nb\nc\n", "a\nB\nc\n", context_len = 3, a_name = "old.txt", b_name = "new.txt", expected = + "--- old.txt\n" + + "+++ new.txt\n" + + "@@ -1,3 +1,3 @@\n" + + " a\n" + + "-b\n" + + "+B\n" + + " c\n") + + // No changes, no output at all + expect_unified(t, "a\nb\n", "a\nb\n", "") + expect_unified(t, "", "", "") +} + +// An empty range starts at the line before it, and a count of 1 is left out. +@(test) +test_unified_ranges :: proc(t: ^testing.T) { + expect_unified(t, "", "x\ny\n", + "--- a\n" + + "+++ b\n" + + "@@ -0,0 +1,2 @@\n" + + "+x\n" + + "+y\n") + + expect_unified(t, "x\n", "", + "--- a\n" + + "+++ b\n" + + "@@ -1 +0,0 @@\n" + + "-x\n") + + expect_unified(t, "a\nb\n", "a\nx\nb\n", context_len = 0, expected = + "--- a\n" + + "+++ b\n" + + "@@ -1,0 +2 @@\n" + + "+x\n") + + expect_unified(t, "a\nx\nb\n", "a\nb\n", context_len = 0, expected = + "--- a\n" + + "+++ b\n" + + "@@ -2 +1,0 @@\n" + + "-x\n") +} + +@(test) +test_unified_no_newline :: proc(t: ^testing.T) { + expect_unified(t, "a\nb", "a\nc", + "--- a\n" + + "+++ b\n" + + "@@ -1,2 +1,2 @@\n" + + " a\n" + + "-b\n" + + "\\ No newline at end of file\n" + + "+c\n" + + "\\ No newline at end of file\n") + + // Adding the final newline changes the line + expect_unified(t, "a", "a\n", + "--- a\n" + + "+++ b\n" + + "@@ -1 +1 @@\n" + + "-a\n" + + "\\ No newline at end of file\n" + + "+a\n") + + // A context line can lack it too + expect_unified(t, "x\ny", "z\ny", + "--- a\n" + + "+++ b\n" + + "@@ -1,2 +1,2 @@\n" + + "-x\n" + + "+z\n" + + " y\n" + + "\\ No newline at end of file\n") +} + +@(test) +test_unified_hunks :: proc(t: ^testing.T) { + a := "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n" + b := "1\ntwo\n3\n4\n5\n6\n7\n8\nnine\n10\n" + + // Six lines apart: separate with a context of 1, one hunk with a context of 3 + expect_unified(t, a, b, context_len = 1, expected = + "--- a\n" + + "+++ b\n" + + "@@ -1,3 +1,3 @@\n" + + " 1\n" + + "-2\n" + + "+two\n" + + " 3\n" + + "@@ -8,3 +8,3 @@\n" + + " 8\n" + + "-9\n" + + "+nine\n" + + " 10\n") + + ONE_HUNK :: + "--- a\n" + + "+++ b\n" + + "@@ -1,10 +1,10 @@\n" + + " 1\n" + + "-2\n" + + "+two\n" + + " 3\n" + + " 4\n" + + " 5\n" + + " 6\n" + + " 7\n" + + " 8\n" + + "-9\n" + + "+nine\n" + + " 10\n" + expect_unified(t, a, b, context_len = 3, expected = ONE_HUNK) + + // All the context there is, asked for as `max(int)` + expect_unified(t, a, b, context_len = max(int), expected = ONE_HUNK) +} + +@(test) +test_unified_style :: proc(t: ^testing.T) { + expect_unified(t, "a\nb\nc\n", "a\nB\nc\n", style = diff.STYLE_ANSI, expected = + "\x1b[1m--- a\x1b[0m\n" + + "\x1b[1m+++ b\x1b[0m\n" + + "\x1b[36m@@ -1,3 +1,3 @@\x1b[0m\n" + + " a\n" + + "\x1b[31m-b\x1b[0m\n" + + "\x1b[32m+B\x1b[0m\n" + + " c\n") +} + +@(test) +test_unified_builder_appends :: proc(t: ^testing.T) { + d := must(diff.diff_text("a\n", "b\n", context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + + sb := strings.builder_make() + defer strings.builder_destroy(&sb) + strings.write_string(&sb, "header\n") + testing.expect_value(t, diff.write_unified_to_builder(&sb, d), nil) + testing.expect_value(t, strings.to_string(sb), "header\n--- a\n+++ b\n@@ -1 +1 @@\n-a\n+b\n") + + // A fixed buffer that is too small fails rather than writing part of the diff, however it is sized + for sizing in diff.Sizing { + small: [8]byte + fixed := strings.builder_from_bytes(small[:]) + testing.expectf(t, diff.write_unified_to_builder(&fixed, d, sizing = sizing) != nil, "%v: a full fixed buffer did not fail", sizing) + testing.expectf(t, len(fixed.buf) == 0, "%v: %d bytes were left in the fixed buffer", sizing, len(fixed.buf)) + } + + // Growing appends too + grown := strings.builder_make() + defer strings.builder_destroy(&grown) + strings.write_string(&grown, "header\n") + testing.expect_value(t, diff.write_unified_to_builder(&grown, d, sizing = .Grow), nil) + testing.expect_value(t, strings.to_string(grown), "header\n--- a\n+++ b\n@@ -1 +1 @@\n-a\n+b\n") +} + +// `.Grow` fails cleanly when the allocator runs out partway, and has nothing to allocate for no changes. +@(test) +test_unified_grow :: proc(t: ^testing.T) { + d := must(diff.diff_text("a\nb\nc\n", "a\nB\nc\n", context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + + backing: [24]byte + arena: mem.Arena + mem.arena_init(&arena, backing[:]) + s, err := diff.unified(d, mem.arena_allocator(&arena), sizing = .Grow) + testing.expect_value(t, err, mem.Allocator_Error.Out_Of_Memory) + testing.expect_value(t, s, "") + + // Nothing to write allocates nothing, so an allocator that cannot free is fine + same := must(diff.diff_text("a\n", "a\n", context.allocator, context.allocator)) + defer diff.text_diff_destroy(same, context.allocator) + mem.arena_free_all(&arena) + empty, empty_err := diff.unified(same, mem.arena_allocator(&arena), sizing = .Grow) + testing.expect_value(t, empty_err, nil) + testing.expect_value(t, empty, "") + testing.expect_value(t, arena.offset, 0) +} + +// On an allocator that resizes its last allocation in place, `.Grow` leaves exactly the output in use. +@(test) +test_unified_grow_in_place :: proc(t: ^testing.T) { + rand.reset(t.seed) + a := random_text(2_000) + b := mutate_lines(a, 200) + d := must(diff.diff_text(a, b, context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + exact := must(diff.unified(d, context.allocator)) + defer delete(exact) + + backing := make([]byte, 4*len(exact) + 1024) + defer delete(backing) + arena: mem.Arena + mem.arena_init(&arena, backing) + + grown := must(diff.unified(d, mem.arena_allocator(&arena), sizing = .Grow)) + testing.expect_value(t, grown, exact) + testing.expect_value(t, arena.offset, len(exact)) +} + +// Takes writes until `limit` bytes are in, then fails every one after, counting them. +Failing_Writer :: struct { + limit, written: int, + failed: bool, + writes_after: int, +} + +failing_stream_proc :: proc(stream_data: rawptr, mode: io.Stream_Mode, p: []byte, offset: i64, whence: io.Seek_From) -> (n: i64, err: io.Error) { + w := (^Failing_Writer)(stream_data) + #partial switch mode { + case .Write: + if w.failed { + w.writes_after += 1 + } + if w.written + len(p) > w.limit { + w.failed = true + return 0, .Short_Write + } + w.written += len(p) + return i64(len(p)), nil + case .Query: + return io.query_utility({.Write, .Query}) + } + return 0, .Empty +} + +// A writer that fails at any byte makes `write_unified` return its error and stop writing, whatever is being written. +@(test) +test_unified_writer_error :: proc(t: ^testing.T) { + // A changed line for emphasis, and a last line without a newline + d := must(diff.diff_text("a\nthe quick fox\nc\nd", "a\nthe slow fox\nc\ne", context.allocator, context.allocator)) + defer diff.text_diff_destroy(d, context.allocator) + + for emphasis in diff.Emphasis { + scratch := make([]byte, diff.emphasis_scratch_size(d, emphasis), context.temp_allocator) + size := diff.unified_size(d, style = diff.STYLE_ANSI, emphasis = emphasis, scratch = scratch) + for limit in 0..=size { + fw := Failing_Writer{limit = limit} + err := diff.write_unified({failing_stream_proc, &fw}, d, style = diff.STYLE_ANSI, emphasis = emphasis, scratch = scratch) + if limit == size { + testing.expectf(t, err == nil && !fw.failed, "%v: the full size failed with %v", emphasis, err) + continue + } + if !testing.expectf(t, err == .Short_Write, "%v: failing after %d of %d bytes returned %v", emphasis, limit, size, err) || + !testing.expectf(t, fw.writes_after == 0, "%v: failing after %d bytes, %d more writes were tried", emphasis, limit, fw.writes_after) { + return + } + } + } +} + +@(test) +test_unified_random :: proc(t: ^testing.T) { + rand.reset(t.seed) + for _ in 0..<150 { + a := random_text(rand.int_max(60)) + b := mutate_lines(a, rand.int_max(15)) + for context_len in HUNK_CONTEXTS { + if !check_unified(t, a, b, context_len) { + return + } + } + if !check_unified(t, a, b, 3, {.Ignore_All_Space, .Ignore_Case}) { + return + } + } +} + +@(test) +test_unified_large :: proc(t: ^testing.T) { + rand.reset(t.seed) + a := random_text(5_000) + b := mutate_lines(a, 300) + check_unified(t, a, b, 3) + check_unified(t, a, b, 0) +} diff --git a/tests/core/io/test_core_io.odin b/tests/core/io/test_core_io.odin index cb40f4728..05dec5d22 100644 --- a/tests/core/io/test_core_io.odin +++ b/tests/core/io/test_core_io.odin @@ -695,3 +695,146 @@ test_bufio_buffered_read_writer :: proc(t: ^testing.T) { log.debugf("%#v", results) } + +@test +test_counting_stream :: proc(t: ^testing.T) { + buf: [32]u8 + for i in 0..