XML hot path: Instrumentation and some optimizations

This commit is contained in:
bplu4t2f committed 2026-10-09 13:11:08 +02:00
1 parent a252797538
commit f25fc5b2a8
3 files changed
+345 -47

No files matched your search

+100 -20
View File
@@ -25,6 +25,8 @@ import "base:runtime"
import "core:unicode/utf8"
import "core:unicode"
import "core:strings"
import "core:time"
import "core:log"
MAX_RUNE_CODEPOINT :: int(unicode.MAX_RUNE)
@@ -78,16 +80,75 @@ XML_Decode_Option :: enum u8 {
}
XML_Decode_Options :: bit_set[XML_Decode_Option; u8]
_Perf_Meas :: struct {
sw: map[_Perf]time.Stopwatch,
}
_Perf :: enum {
set_context,
init_builder,
advance,
cdata,
decode_amp,
av_normalization,
line_ends,
}
_meas_reset :: proc(m: ^_Perf_Meas) {
m.sw = make(map[_Perf]time.Stopwatch, context.temp_allocator)
for p in _Perf {
m.sw[p] = {}
}
}
_meas_begin :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
time.stopwatch_start(&m.sw[which])
}
_meas_end :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
time.stopwatch_stop(&m.sw[which])
}
_meas_log :: proc(m: _Perf_Meas) {
for p in _Perf {
sw := m.sw[p]
log.infof("% -35v: %v", p, time.stopwatch_duration(sw))
}
}
@(thread_local) _decode_xml_meas: _Perf_Meas
// Decode a string that may include SGML/XML/HTML entities.
// The caller has to free the result.
decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := context.allocator) -> (decoded: string, err: Error) {
builder := strings.builder_make(allocator)
defer strings.builder_destroy(&builder)
decoded, err = decode_xml_sb(&builder, input, options, allocator)
decoded = strings.clone(decoded, allocator)
return
}
decode_xml_sb :: proc(builder: ^strings.Builder, input: string, options := XML_Decode_Options{}, allocator := context.allocator) -> (decoded: string, err: Error) {
// PERF: Hot
m := &_decode_xml_meas
_meas_begin(m, .set_context)
context.allocator = allocator
_meas_end(m, .set_context)
l := len(input)
if l == 0 { return "", .None }
builder := strings.builder_make()
defer strings.builder_destroy(&builder)
// PERF: This is a hot path, using an arena allocator here is beneficial.
//builder, builder_err := strings.builder_make_len_cap(0, 250, context.temp_allocator)
//if builder_err != nil {
// Fallback to the provided allocator if no temp allocator is available.
//_meas_begin(m, .init_builder)
// builder := strings.builder_make_len_cap(0, 100)
//_meas_end(m, .init_builder)
//}
//defer strings.builder_destroy(&builder)
string_start_in_builder := len(builder.buf)
t := Tokenizer{src=input}
in_data := false
@@ -95,7 +156,10 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator :=
prev: rune = ' '
loop: for {
_meas_begin(m, .advance)
advance(&t) or_return
_meas_end(m, .advance)
if t.r < 0 { break loop }
// Below here we're never inside a CDATA tag. At most we'll see the start of one,
@@ -111,40 +175,46 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator :=
Keep in mind that we could already *be* inside a CDATA tag.
If so, write `<` as a literal and continue.
*/
_meas_begin(m, .cdata)
if in_data {
write_rune(&builder, '<')
write_rune(builder, '<')
continue
}
in_data = _handle_xml_special(&t, &builder, options) or_return
in_data = _handle_xml_special(&t, builder, options) or_return
_meas_end(m, .cdata)
case ']':
// If we're unboxing _and_ decoding CDATA, we'll have to check for the end tag.
if in_data {
_meas_begin(m, .cdata)
if strings.has_prefix(t.src[t.offset:], CDATA_END) {
in_data = false
t.read_offset += len(CDATA_END) - 1
}
_meas_end(m, .cdata)
continue
} else {
write_rune(&builder, ']')
write_rune(builder, ']')
}
case:
if in_data && .Decode_CDATA not_in options {
// Unboxed, but undecoded.
write_rune(&builder, t.r)
_meas_begin(m, .cdata)
write_rune(builder, t.r)
_meas_end(m, .cdata)
continue
}
if t.r == '&' {
if entity, entity_err := _extract_xml_entity(&t); entity_err != .None {
// We read to the end of the string without closing the entity. Pass through as-is.
write_string(&builder, entity)
} else {
_meas_begin(m, .decode_amp)
defer _meas_end(m, .decode_amp)
entity, entity_err := _extract_xml_entity(&t)
if entity_err == nil {
if .No_Entity_Decode not_in options {
if decoded, count, ok := xml_decode_entity(entity); ok {
for i in 0..<count {
write_rune(&builder, decoded[i])
write_rune(builder, decoded[i])
}
prev = decoded[count - 1]
continue
@@ -152,41 +222,51 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator :=
}
// Literal passthrough because the decode failed or we want entities not decoded.
write_string(&builder, "&")
write_string(&builder, entity)
write_string(&builder, ";")
// PERF: Cold
write_string(builder, "&")
write_string(builder, entity)
write_string(builder, ";")
} else {
// PERF: Cold
// We read to the end of the string without closing the entity. Pass through as-is.
write_string(builder, entity)
}
} else {
// Handle AV Normalization: https://www.w3.org/TR/2006/REC-xml11-20060816/#AVNormalize
if .Normalize_Whitespace in options {
_meas_begin(m, .av_normalization)
switch t.r {
case ' ', '\r', '\n', '\t':
if prev != ' ' {
write_rune(&builder, ' ')
write_rune(builder, ' ')
prev = ' '
}
case:
write_rune(&builder, t.r)
write_rune(builder, t.r)
prev = t.r
}
_meas_end(m, .av_normalization)
} else {
// https://www.w3.org/TR/2006/REC-xml11-20060816/#sec-line-ends
_meas_begin(m, .line_ends)
switch t.r {
case '\n', 0x85, 0x2028:
write_rune(&builder, '\n')
write_rune(builder, '\n')
case '\r': // Do nothing until next character
case:
if prev == '\r' { // Turn a single carriage return into a \n
write_rune(&builder, '\n')
write_rune(builder, '\n')
}
write_rune(&builder, t.r)
write_rune(builder, t.r)
}
prev = t.r
_meas_end(m, .line_ends)
}
}
}
}
return strings.clone(strings.to_string(builder), allocator), err
//return strings.clone(strings.to_string(builder), allocator), err
return transmute(string)builder.buf[string_start_in_builder:], err
}
advance :: proc(t: ^Tokenizer) -> (err: Error) {
+96 -15
View File
@@ -129,6 +129,8 @@ error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
@(optimization_mode="favor_size")
advance_rune :: proc(t: ^Tokenizer) {
//m := &_parse_bytes_meas
//_meas_begin(m, .advance_rune_total)
#no_bounds_check {
/*
Already bounds-checked here.
@@ -162,6 +164,7 @@ advance_rune :: proc(t: ^Tokenizer) {
t.ch = -1
}
}
//_meas_end(m, .advance_rune_total)
}
peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte {
@@ -172,25 +175,94 @@ peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte {
}
@(optimization_mode="favor_size")
// PERF: warm
skip_whitespace :: proc(t: ^Tokenizer) {
for {
switch t.ch {
case ' ', '\t', '\r', '\n':
advance_rune(t)
case:
return
//m := &_parse_bytes_meas
//_meas_begin(m, .skip_whitespace_total)
/*
Already bounds-checked here.
*/
/*#no_bounds_check*/ {
loop: for {
/*if t.ch == ' ' || t.ch == '\t' || t.ch == '\r' {
// PERF: Hot.
if t.read_offset < len(t.src) {
t.offset = t.read_offset
c := t.src[t.read_offset]
if c < utf8.RUNE_SELF {
t.read_offset += 1
t.ch = cast(rune)c
} else {
// PERF: Unlikely.
advance_rune(t)
}
} else {
// PERF: Unlikely.
advance_rune(t)
}
} else if t.ch == '\n' {
// PERF: Hot.
if t.read_offset < len(t.src) {
t.offset = t.read_offset
t.line_offset = t.offset
t.line_count += 1
c := t.src[t.read_offset]
if c < utf8.RUNE_SELF {
t.read_offset += 1
t.ch = cast(rune)c
} else {
// PERF: Unlikely.
advance_rune(t)
}
} else {
// PERF: Unlikely.
advance_rune(t)
}
} else {
break
}*/
switch t.ch {
case ' ', '\t', '\r', '\n':
advance_rune(t)
case:
break loop
}
}
}
//_meas_end(m, .skip_whitespace_total)
}
_is_letter_map := _is_letter_map_init()
_is_letter_map_init :: proc "contextless" () -> [0x80]bool {
m: [0x80]bool
assert_contextless(utf8.RUNE_SELF == len(m))
m['_'] = true
for c in 'A' ..= 'Z' {
m[c] = true
}
for c in 'a' ..= 'z' {
m[c] = true
}
return m
}
@(optimization_mode="favor_size")
is_letter :: proc(r: rune) -> bool {
// PERF: Warm
if r < utf8.RUNE_SELF {
//#no_bounds_check {
// if _is_letter_map[r] {
// return true
// }
//}
switch r {
case '_':
return true
case 'A'..='Z', 'a'..='z':
return true
case '_':
return true
}
}
return unicode.is_letter(r)
@@ -294,6 +366,10 @@ skip_cdata :: proc(t: ^Tokenizer) -> (err: Error) {
@(optimization_mode="favor_size")
scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false, multiline := true) -> (value: string, err: Error) {
//m := &_parse_bytes_meas
//_meas_begin(m, .scan_string_total)
//defer _meas_end(m, .scan_string_total)
err = .None
loop: for {
@@ -352,14 +428,18 @@ scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close
return lit, err
}
peek :: proc(t: ^Tokenizer) -> (token: Token) {
old := t^
token = scan(t)
t^ = old
return token
peek :: #force_inline proc(t: ^Tokenizer) -> (token: Token) {
m := &_parse_bytes_meas
_meas_begin(m, .peek_total)
tmp := t^
r := scan(&tmp)
_meas_end(m, .peek_total)
return r
}
scan :: proc(t: ^Tokenizer, multiline_string := false) -> Token {
m := &_parse_bytes_meas
_meas_begin(m, .scan_total)
skip_whitespace(t)
offset := t.offset
@@ -367,10 +447,10 @@ scan :: proc(t: ^Tokenizer, multiline_string := false) -> Token {
kind: Token_Kind
err: Error
lit: string
pos := offset_to_pos(t, offset)
pos := #force_inline offset_to_pos(t, offset)
switch ch := t.ch; true {
case is_letter(ch):
case #force_inline is_letter(ch):
lit = scan_identifier(t)
kind = .Ident
@@ -411,5 +491,6 @@ scan :: proc(t: ^Tokenizer, multiline_string := false) -> Token {
if kind != .String && lit == "" {
lit = string(t.src[offset : t.offset])
}
_meas_end(m, .scan_total)
return Token{kind, lit, pos}
}
+149 -12
View File
@@ -15,6 +15,8 @@ import "core:encoding/entity"
import "base:intrinsics"
import "core:mem"
import "core:strings"
import "core:time"
import "core:log"
likely :: intrinsics.expect
@@ -163,15 +165,79 @@ Error :: enum {
Conflicting_Options,
}
_Perf_Meas :: struct {
sw: map[_Perf]time.Stopwatch,
}
_Perf :: enum {
clone_input,
elements,
attributes,
attributes_set_context,
attributes_peek,
attributes_parse_attribute,
attributes_check_duplicates,
attributes_append,
attributes_skip_whitespace,
text_content,
cdata,
comments,
resize_result,
peek_total,
scan_total,
scan_string_total,
skip_whitespace_total,
advance_rune_total,
parse_attribute_set_context,
parse_attribute_tokenize_ident,
parse_attribute_tokenize_eq,
parse_attribute_tokenize_value,
parse_attribute_decode,
parse_attribute_append,
}
_meas_reset :: proc(m: ^_Perf_Meas) {
m.sw = make(map[_Perf]time.Stopwatch, context.temp_allocator)
for p in _Perf {
m.sw[p] = {}
}
}
_meas_begin :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
time.stopwatch_start(&m.sw[which])
}
_meas_end :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
time.stopwatch_stop(&m.sw[which])
}
_meas_log :: proc(m: _Perf_Meas) {
for p in _Perf {
sw := m.sw[p]
log.infof("% -35v: %v", p, time.stopwatch_duration(sw))
}
}
@(thread_local) _parse_bytes_meas: _Perf_Meas
parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) {
data := data
context.allocator = allocator
opts := validate_options(options) or_return
m := &_parse_bytes_meas
_meas_reset(m)
defer _meas_log(m^)
entity._meas_reset(&entity._decode_xml_meas)
defer entity._meas_log(entity._decode_xml_meas)
// If `.Input_May_Be_Modified` is not specified, we duplicate the input so that we can modify it in-place.
if .Input_May_Be_Modified not_in opts.flags {
_meas_begin(m, .clone_input)
data = bytes.clone(data)
_meas_end(m, .clone_input)
}
t := new(Tokenizer)
@@ -203,6 +269,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
// NOTE(Jeroen): We're not using a switch because this if-else chain ordered by likelihood is 2.5% faster at -o:size and -o:speed.
if likely(open.kind, Token_Kind.Ident) == .Ident {
// e.g. <odin - Start of new element.
_meas_begin(m, .elements)
element = new_element(doc)
if element == 0 { // First Element
parent = element
@@ -212,8 +279,14 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
doc.elements[element].parent = parent
doc.elements[element].ident = open.text
_meas_end(m, .elements)
_meas_begin(m, .attributes)
// PERF: Hot !!
parse_attributes(doc, &doc.elements[element].attribs) or_return
_meas_end(m, .attributes)
_meas_begin(m, .elements)
// If a DOCTYPE is present _or_ the caller
// asked for a specific DOCTYPE and the DOCTYPE
@@ -245,8 +318,11 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
return
}
_meas_end(m, .elements)
} else if open.kind == .Slash {
// Close tag.
_meas_begin(m, .elements)
ident := expect(t, .Ident) or_return
_ = expect(t, .Gt) or_return
@@ -256,6 +332,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
}
parent = doc.elements[element].parent
element = parent
_meas_end(m, .elements)
} else if open.kind == .Exclaim {
// <!
@@ -290,6 +367,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
// Comment: <!-- -->.
// The grammar does not allow a comment to end in --->
expect(t, .Dash)
_meas_begin(m, .comments)
comment := scan_comment(t) or_return
if .Intern_Comments in opts.flags {
@@ -303,6 +381,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
append(&doc.elements[element].value, el)
}
}
_meas_end(m, .comments)
case .Open_Bracket:
// This could be a CDATA tag part of a tag's body. Unread the `<![`
@@ -312,7 +391,9 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
// Instead of calling `parse_body` here, we could also `continue loop`
// and fall through to the `case:` at the bottom of the outer loop.
// This makes the intent clearer.
_meas_begin(m, .cdata)
parse_body(doc, element, opts) or_return
_meas_end(m, .cdata)
case:
error(t, t.offset, "Unexpected Token after <!: %#v", next)
@@ -348,7 +429,9 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
case:
// This should be a tag's body text.
_meas_begin(m, .text_content)
parse_body(doc, element, opts) or_return
_meas_end(m, .text_content)
}
}
@@ -360,7 +443,9 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
return doc, .No_DocType
}
_meas_begin(m, .resize_result)
resize(&doc.elements, int(doc.element_count))
_meas_end(m, .resize_result)
return doc, .None
}
@@ -408,7 +493,7 @@ validate_options :: proc(options: Options) -> (validated: Options, err: Error) {
return validated, .None
}
expect :: proc(t: ^Tokenizer, kind: Token_Kind, multiline_string := false) -> (tok: Token, err: Error) {
expect :: #force_inline proc(t: ^Tokenizer, kind: Token_Kind, multiline_string := false) -> (tok: Token, err: Error) {
tok = scan(t, multiline_string=multiline_string)
if tok.kind == kind { return tok, .None }
@@ -416,18 +501,34 @@ expect :: proc(t: ^Tokenizer, kind: Token_Kind, multiline_string := false) -> (t
return tok, .Unexpected_Token
}
parse_attribute :: proc(doc: ^Document) -> (attr: Attribute, offset: int, err: Error) {
_parse_attribute :: proc(doc: ^Document, builder: ^strings.Builder) -> (attr: Attribute, offset: int, err: Error) {
assert(doc != nil)
m := &_parse_bytes_meas
_meas_begin(m, .parse_attribute_set_context)
context.allocator = doc.allocator
_meas_end(m, .parse_attribute_set_context)
t := doc.tokenizer
_meas_begin(m, .parse_attribute_tokenize_ident)
key := expect(t, .Ident) or_return
_ = expect(t, .Eq) or_return
value := expect(t, .String, multiline_string=true) or_return
_meas_end(m, .parse_attribute_tokenize_ident)
normalized, normalize_err := entity.decode_xml(value.text, {.Normalize_Whitespace}, doc.allocator)
_meas_begin(m, .parse_attribute_tokenize_eq)
_ = expect(t, .Eq) or_return
_meas_end(m, .parse_attribute_tokenize_eq)
_meas_begin(m, .parse_attribute_tokenize_value)
value := expect(t, .String, multiline_string=true) or_return
_meas_end(m, .parse_attribute_tokenize_value)
_meas_begin(m, .parse_attribute_decode)
//builder := strings.builder_make_len_cap(0, len(value.text) + 10, doc.allocator)
normalized, normalize_err := entity.decode_xml_sb(builder, value.text, {.Normalize_Whitespace}, doc.allocator)
_meas_end(m, .parse_attribute_decode)
if normalize_err == .None {
append(&doc.strings_to_free, normalized)
//_meas_begin(m, .parse_attribute_append)
//append(&doc.strings_to_free, strings.to_string(builder))
//_meas_end(m, .parse_attribute_append)
value.text = normalized
}
@@ -445,20 +546,56 @@ check_duplicate_attributes :: proc(t: ^Tokenizer, attribs: Attributes, attr: Att
return .Duplicate_Attribute
}
}
return .None
return .None
}
parse_attributes :: proc(doc: ^Document, attribs: ^Attributes) -> (err: Error) {
assert(doc != nil)
context.allocator = doc.allocator
m := &_parse_bytes_meas
_meas_begin(m, .attributes_set_context)
context.allocator = doc.allocator // PERF: mildly warmish
_meas_end(m, .attributes_set_context)
t := doc.tokenizer
for peek(t).kind == .Ident {
attr, offset := parse_attribute(doc) or_return
check_duplicate_attributes(t, attribs^, attr, offset) or_return
append(attribs, attr)
// PERF: Hot path. Consolidate attribute strings into one buffer.
// !!!!!!!!!!!!!!!!!
// TODO: Cannot leave it like this !! This will cause strings to become invalid in case it ever needs to grow the buffer !!
// !!!!!!!!!!!!!!!!!
builder := strings.builder_make_len_cap(0, 500, doc.allocator)
defer {
append(&doc.strings_to_free, strings.to_string(builder))
}
for {
_meas_begin(m, .attributes_peek)
k := peek(t).kind
_meas_end(m, .attributes_peek)
if k != .Ident {
break
}
_meas_begin(m, .attributes_parse_attribute)
attr, offset := _parse_attribute(doc, &builder) or_return
_meas_end(m, .attributes_parse_attribute)
_meas_begin(m, .attributes_check_duplicates)
check_duplicate_attributes(t, attribs^, attr, offset) or_return
_meas_end(m, .attributes_check_duplicates)
_meas_begin(m, .attributes_append)
append(attribs, attr)
_meas_end(m, .attributes_append)
}
//log.infof("Builder contents: %v", strings.to_string(builder))
//for attr in attribs {
// log.infof("Attr: %v", attr.val)
//}
_meas_begin(m, .attributes_skip_whitespace)
skip_whitespace(t)
_meas_end(m, .attributes_skip_whitespace)
return .None
}