mirror of
https://github.com/odin-lang/Odin.git
synced 2026-10-12 11:09:25 -04:00
XML hot path: Instrumentation and some optimizations
This commit is contained in:
1 parent
a252797538
commit
f25fc5b2a8
3 files changed
+345
-47
No files matched your search
@@ -25,6 +25,8 @@ import "base:runtime"
|
||||
import "core:unicode/utf8"
|
||||
import "core:unicode"
|
||||
import "core:strings"
|
||||
import "core:time"
|
||||
import "core:log"
|
||||
|
||||
MAX_RUNE_CODEPOINT :: int(unicode.MAX_RUNE)
|
||||
|
||||
@@ -78,16 +80,75 @@ XML_Decode_Option :: enum u8 {
|
||||
}
|
||||
XML_Decode_Options :: bit_set[XML_Decode_Option; u8]
|
||||
|
||||
_Perf_Meas :: struct {
|
||||
sw: map[_Perf]time.Stopwatch,
|
||||
}
|
||||
|
||||
_Perf :: enum {
|
||||
set_context,
|
||||
init_builder,
|
||||
advance,
|
||||
cdata,
|
||||
decode_amp,
|
||||
av_normalization,
|
||||
line_ends,
|
||||
}
|
||||
|
||||
_meas_reset :: proc(m: ^_Perf_Meas) {
|
||||
m.sw = make(map[_Perf]time.Stopwatch, context.temp_allocator)
|
||||
for p in _Perf {
|
||||
m.sw[p] = {}
|
||||
}
|
||||
}
|
||||
|
||||
_meas_begin :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
|
||||
time.stopwatch_start(&m.sw[which])
|
||||
}
|
||||
|
||||
_meas_end :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
|
||||
time.stopwatch_stop(&m.sw[which])
|
||||
}
|
||||
|
||||
_meas_log :: proc(m: _Perf_Meas) {
|
||||
for p in _Perf {
|
||||
sw := m.sw[p]
|
||||
log.infof("% -35v: %v", p, time.stopwatch_duration(sw))
|
||||
}
|
||||
}
|
||||
|
||||
@(thread_local) _decode_xml_meas: _Perf_Meas
|
||||
|
||||
// Decode a string that may include SGML/XML/HTML entities.
|
||||
// The caller has to free the result.
|
||||
decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := context.allocator) -> (decoded: string, err: Error) {
|
||||
builder := strings.builder_make(allocator)
|
||||
defer strings.builder_destroy(&builder)
|
||||
decoded, err = decode_xml_sb(&builder, input, options, allocator)
|
||||
decoded = strings.clone(decoded, allocator)
|
||||
return
|
||||
}
|
||||
|
||||
decode_xml_sb :: proc(builder: ^strings.Builder, input: string, options := XML_Decode_Options{}, allocator := context.allocator) -> (decoded: string, err: Error) {
|
||||
// PERF: Hot
|
||||
m := &_decode_xml_meas
|
||||
_meas_begin(m, .set_context)
|
||||
context.allocator = allocator
|
||||
_meas_end(m, .set_context)
|
||||
|
||||
l := len(input)
|
||||
if l == 0 { return "", .None }
|
||||
|
||||
builder := strings.builder_make()
|
||||
defer strings.builder_destroy(&builder)
|
||||
// PERF: This is a hot path, using an arena allocator here is beneficial.
|
||||
//builder, builder_err := strings.builder_make_len_cap(0, 250, context.temp_allocator)
|
||||
//if builder_err != nil {
|
||||
// Fallback to the provided allocator if no temp allocator is available.
|
||||
//_meas_begin(m, .init_builder)
|
||||
// builder := strings.builder_make_len_cap(0, 100)
|
||||
//_meas_end(m, .init_builder)
|
||||
//}
|
||||
//defer strings.builder_destroy(&builder)
|
||||
|
||||
string_start_in_builder := len(builder.buf)
|
||||
|
||||
t := Tokenizer{src=input}
|
||||
in_data := false
|
||||
@@ -95,7 +156,10 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator :=
|
||||
prev: rune = ' '
|
||||
|
||||
loop: for {
|
||||
_meas_begin(m, .advance)
|
||||
advance(&t) or_return
|
||||
_meas_end(m, .advance)
|
||||
|
||||
if t.r < 0 { break loop }
|
||||
|
||||
// Below here we're never inside a CDATA tag. At most we'll see the start of one,
|
||||
@@ -111,40 +175,46 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator :=
|
||||
Keep in mind that we could already *be* inside a CDATA tag.
|
||||
If so, write `<` as a literal and continue.
|
||||
*/
|
||||
_meas_begin(m, .cdata)
|
||||
if in_data {
|
||||
write_rune(&builder, '<')
|
||||
write_rune(builder, '<')
|
||||
continue
|
||||
}
|
||||
in_data = _handle_xml_special(&t, &builder, options) or_return
|
||||
in_data = _handle_xml_special(&t, builder, options) or_return
|
||||
_meas_end(m, .cdata)
|
||||
|
||||
case ']':
|
||||
// If we're unboxing _and_ decoding CDATA, we'll have to check for the end tag.
|
||||
if in_data {
|
||||
_meas_begin(m, .cdata)
|
||||
if strings.has_prefix(t.src[t.offset:], CDATA_END) {
|
||||
in_data = false
|
||||
t.read_offset += len(CDATA_END) - 1
|
||||
}
|
||||
_meas_end(m, .cdata)
|
||||
continue
|
||||
} else {
|
||||
write_rune(&builder, ']')
|
||||
write_rune(builder, ']')
|
||||
}
|
||||
|
||||
case:
|
||||
if in_data && .Decode_CDATA not_in options {
|
||||
// Unboxed, but undecoded.
|
||||
write_rune(&builder, t.r)
|
||||
_meas_begin(m, .cdata)
|
||||
write_rune(builder, t.r)
|
||||
_meas_end(m, .cdata)
|
||||
continue
|
||||
}
|
||||
|
||||
if t.r == '&' {
|
||||
if entity, entity_err := _extract_xml_entity(&t); entity_err != .None {
|
||||
// We read to the end of the string without closing the entity. Pass through as-is.
|
||||
write_string(&builder, entity)
|
||||
} else {
|
||||
_meas_begin(m, .decode_amp)
|
||||
defer _meas_end(m, .decode_amp)
|
||||
entity, entity_err := _extract_xml_entity(&t)
|
||||
if entity_err == nil {
|
||||
if .No_Entity_Decode not_in options {
|
||||
if decoded, count, ok := xml_decode_entity(entity); ok {
|
||||
for i in 0..<count {
|
||||
write_rune(&builder, decoded[i])
|
||||
write_rune(builder, decoded[i])
|
||||
}
|
||||
prev = decoded[count - 1]
|
||||
continue
|
||||
@@ -152,41 +222,51 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator :=
|
||||
}
|
||||
|
||||
// Literal passthrough because the decode failed or we want entities not decoded.
|
||||
write_string(&builder, "&")
|
||||
write_string(&builder, entity)
|
||||
write_string(&builder, ";")
|
||||
// PERF: Cold
|
||||
write_string(builder, "&")
|
||||
write_string(builder, entity)
|
||||
write_string(builder, ";")
|
||||
} else {
|
||||
// PERF: Cold
|
||||
// We read to the end of the string without closing the entity. Pass through as-is.
|
||||
write_string(builder, entity)
|
||||
}
|
||||
} else {
|
||||
// Handle AV Normalization: https://www.w3.org/TR/2006/REC-xml11-20060816/#AVNormalize
|
||||
if .Normalize_Whitespace in options {
|
||||
_meas_begin(m, .av_normalization)
|
||||
switch t.r {
|
||||
case ' ', '\r', '\n', '\t':
|
||||
if prev != ' ' {
|
||||
write_rune(&builder, ' ')
|
||||
write_rune(builder, ' ')
|
||||
prev = ' '
|
||||
}
|
||||
case:
|
||||
write_rune(&builder, t.r)
|
||||
write_rune(builder, t.r)
|
||||
prev = t.r
|
||||
}
|
||||
_meas_end(m, .av_normalization)
|
||||
} else {
|
||||
// https://www.w3.org/TR/2006/REC-xml11-20060816/#sec-line-ends
|
||||
_meas_begin(m, .line_ends)
|
||||
switch t.r {
|
||||
case '\n', 0x85, 0x2028:
|
||||
write_rune(&builder, '\n')
|
||||
write_rune(builder, '\n')
|
||||
case '\r': // Do nothing until next character
|
||||
case:
|
||||
if prev == '\r' { // Turn a single carriage return into a \n
|
||||
write_rune(&builder, '\n')
|
||||
write_rune(builder, '\n')
|
||||
}
|
||||
write_rune(&builder, t.r)
|
||||
write_rune(builder, t.r)
|
||||
}
|
||||
prev = t.r
|
||||
_meas_end(m, .line_ends)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.clone(strings.to_string(builder), allocator), err
|
||||
//return strings.clone(strings.to_string(builder), allocator), err
|
||||
return transmute(string)builder.buf[string_start_in_builder:], err
|
||||
}
|
||||
|
||||
advance :: proc(t: ^Tokenizer) -> (err: Error) {
|
||||
|
||||
@@ -129,6 +129,8 @@ error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
|
||||
|
||||
@(optimization_mode="favor_size")
|
||||
advance_rune :: proc(t: ^Tokenizer) {
|
||||
//m := &_parse_bytes_meas
|
||||
//_meas_begin(m, .advance_rune_total)
|
||||
#no_bounds_check {
|
||||
/*
|
||||
Already bounds-checked here.
|
||||
@@ -162,6 +164,7 @@ advance_rune :: proc(t: ^Tokenizer) {
|
||||
t.ch = -1
|
||||
}
|
||||
}
|
||||
//_meas_end(m, .advance_rune_total)
|
||||
}
|
||||
|
||||
peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte {
|
||||
@@ -172,25 +175,94 @@ peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte {
|
||||
}
|
||||
|
||||
@(optimization_mode="favor_size")
|
||||
// PERF: warm
|
||||
skip_whitespace :: proc(t: ^Tokenizer) {
|
||||
for {
|
||||
switch t.ch {
|
||||
case ' ', '\t', '\r', '\n':
|
||||
advance_rune(t)
|
||||
case:
|
||||
return
|
||||
//m := &_parse_bytes_meas
|
||||
//_meas_begin(m, .skip_whitespace_total)
|
||||
/*
|
||||
Already bounds-checked here.
|
||||
*/
|
||||
/*#no_bounds_check*/ {
|
||||
loop: for {
|
||||
/*if t.ch == ' ' || t.ch == '\t' || t.ch == '\r' {
|
||||
// PERF: Hot.
|
||||
if t.read_offset < len(t.src) {
|
||||
t.offset = t.read_offset
|
||||
c := t.src[t.read_offset]
|
||||
if c < utf8.RUNE_SELF {
|
||||
t.read_offset += 1
|
||||
t.ch = cast(rune)c
|
||||
} else {
|
||||
// PERF: Unlikely.
|
||||
advance_rune(t)
|
||||
}
|
||||
} else {
|
||||
// PERF: Unlikely.
|
||||
advance_rune(t)
|
||||
}
|
||||
} else if t.ch == '\n' {
|
||||
// PERF: Hot.
|
||||
if t.read_offset < len(t.src) {
|
||||
t.offset = t.read_offset
|
||||
t.line_offset = t.offset
|
||||
t.line_count += 1
|
||||
c := t.src[t.read_offset]
|
||||
if c < utf8.RUNE_SELF {
|
||||
t.read_offset += 1
|
||||
t.ch = cast(rune)c
|
||||
} else {
|
||||
// PERF: Unlikely.
|
||||
advance_rune(t)
|
||||
}
|
||||
} else {
|
||||
// PERF: Unlikely.
|
||||
advance_rune(t)
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}*/
|
||||
|
||||
switch t.ch {
|
||||
case ' ', '\t', '\r', '\n':
|
||||
advance_rune(t)
|
||||
case:
|
||||
break loop
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
//_meas_end(m, .skip_whitespace_total)
|
||||
}
|
||||
|
||||
_is_letter_map := _is_letter_map_init()
|
||||
|
||||
_is_letter_map_init :: proc "contextless" () -> [0x80]bool {
|
||||
m: [0x80]bool
|
||||
assert_contextless(utf8.RUNE_SELF == len(m))
|
||||
m['_'] = true
|
||||
for c in 'A' ..= 'Z' {
|
||||
m[c] = true
|
||||
}
|
||||
for c in 'a' ..= 'z' {
|
||||
m[c] = true
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@(optimization_mode="favor_size")
|
||||
is_letter :: proc(r: rune) -> bool {
|
||||
// PERF: Warm
|
||||
if r < utf8.RUNE_SELF {
|
||||
//#no_bounds_check {
|
||||
// if _is_letter_map[r] {
|
||||
// return true
|
||||
// }
|
||||
//}
|
||||
switch r {
|
||||
case '_':
|
||||
return true
|
||||
case 'A'..='Z', 'a'..='z':
|
||||
return true
|
||||
case '_':
|
||||
return true
|
||||
}
|
||||
}
|
||||
return unicode.is_letter(r)
|
||||
@@ -294,6 +366,10 @@ skip_cdata :: proc(t: ^Tokenizer) -> (err: Error) {
|
||||
|
||||
@(optimization_mode="favor_size")
|
||||
scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false, multiline := true) -> (value: string, err: Error) {
|
||||
//m := &_parse_bytes_meas
|
||||
//_meas_begin(m, .scan_string_total)
|
||||
//defer _meas_end(m, .scan_string_total)
|
||||
|
||||
err = .None
|
||||
|
||||
loop: for {
|
||||
@@ -352,14 +428,18 @@ scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close
|
||||
return lit, err
|
||||
}
|
||||
|
||||
peek :: proc(t: ^Tokenizer) -> (token: Token) {
|
||||
old := t^
|
||||
token = scan(t)
|
||||
t^ = old
|
||||
return token
|
||||
peek :: #force_inline proc(t: ^Tokenizer) -> (token: Token) {
|
||||
m := &_parse_bytes_meas
|
||||
_meas_begin(m, .peek_total)
|
||||
tmp := t^
|
||||
r := scan(&tmp)
|
||||
_meas_end(m, .peek_total)
|
||||
return r
|
||||
}
|
||||
|
||||
scan :: proc(t: ^Tokenizer, multiline_string := false) -> Token {
|
||||
m := &_parse_bytes_meas
|
||||
_meas_begin(m, .scan_total)
|
||||
skip_whitespace(t)
|
||||
|
||||
offset := t.offset
|
||||
@@ -367,10 +447,10 @@ scan :: proc(t: ^Tokenizer, multiline_string := false) -> Token {
|
||||
kind: Token_Kind
|
||||
err: Error
|
||||
lit: string
|
||||
pos := offset_to_pos(t, offset)
|
||||
pos := #force_inline offset_to_pos(t, offset)
|
||||
|
||||
switch ch := t.ch; true {
|
||||
case is_letter(ch):
|
||||
case #force_inline is_letter(ch):
|
||||
lit = scan_identifier(t)
|
||||
kind = .Ident
|
||||
|
||||
@@ -411,5 +491,6 @@ scan :: proc(t: ^Tokenizer, multiline_string := false) -> Token {
|
||||
if kind != .String && lit == "" {
|
||||
lit = string(t.src[offset : t.offset])
|
||||
}
|
||||
_meas_end(m, .scan_total)
|
||||
return Token{kind, lit, pos}
|
||||
}
|
||||
@@ -15,6 +15,8 @@ import "core:encoding/entity"
|
||||
import "base:intrinsics"
|
||||
import "core:mem"
|
||||
import "core:strings"
|
||||
import "core:time"
|
||||
import "core:log"
|
||||
|
||||
likely :: intrinsics.expect
|
||||
|
||||
@@ -163,15 +165,79 @@ Error :: enum {
|
||||
Conflicting_Options,
|
||||
}
|
||||
|
||||
_Perf_Meas :: struct {
|
||||
sw: map[_Perf]time.Stopwatch,
|
||||
}
|
||||
|
||||
_Perf :: enum {
|
||||
clone_input,
|
||||
elements,
|
||||
attributes,
|
||||
attributes_set_context,
|
||||
attributes_peek,
|
||||
attributes_parse_attribute,
|
||||
attributes_check_duplicates,
|
||||
attributes_append,
|
||||
attributes_skip_whitespace,
|
||||
text_content,
|
||||
cdata,
|
||||
comments,
|
||||
resize_result,
|
||||
peek_total,
|
||||
scan_total,
|
||||
scan_string_total,
|
||||
skip_whitespace_total,
|
||||
advance_rune_total,
|
||||
parse_attribute_set_context,
|
||||
parse_attribute_tokenize_ident,
|
||||
parse_attribute_tokenize_eq,
|
||||
parse_attribute_tokenize_value,
|
||||
parse_attribute_decode,
|
||||
parse_attribute_append,
|
||||
}
|
||||
|
||||
_meas_reset :: proc(m: ^_Perf_Meas) {
|
||||
m.sw = make(map[_Perf]time.Stopwatch, context.temp_allocator)
|
||||
for p in _Perf {
|
||||
m.sw[p] = {}
|
||||
}
|
||||
}
|
||||
|
||||
_meas_begin :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
|
||||
time.stopwatch_start(&m.sw[which])
|
||||
}
|
||||
|
||||
_meas_end :: #force_inline proc(m: ^_Perf_Meas, which: _Perf) {
|
||||
time.stopwatch_stop(&m.sw[which])
|
||||
}
|
||||
|
||||
_meas_log :: proc(m: _Perf_Meas) {
|
||||
for p in _Perf {
|
||||
sw := m.sw[p]
|
||||
log.infof("% -35v: %v", p, time.stopwatch_duration(sw))
|
||||
}
|
||||
}
|
||||
|
||||
@(thread_local) _parse_bytes_meas: _Perf_Meas
|
||||
|
||||
parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) {
|
||||
data := data
|
||||
context.allocator = allocator
|
||||
|
||||
opts := validate_options(options) or_return
|
||||
|
||||
m := &_parse_bytes_meas
|
||||
_meas_reset(m)
|
||||
defer _meas_log(m^)
|
||||
|
||||
entity._meas_reset(&entity._decode_xml_meas)
|
||||
defer entity._meas_log(entity._decode_xml_meas)
|
||||
|
||||
// If `.Input_May_Be_Modified` is not specified, we duplicate the input so that we can modify it in-place.
|
||||
if .Input_May_Be_Modified not_in opts.flags {
|
||||
_meas_begin(m, .clone_input)
|
||||
data = bytes.clone(data)
|
||||
_meas_end(m, .clone_input)
|
||||
}
|
||||
|
||||
t := new(Tokenizer)
|
||||
@@ -203,6 +269,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
// NOTE(Jeroen): We're not using a switch because this if-else chain ordered by likelihood is 2.5% faster at -o:size and -o:speed.
|
||||
if likely(open.kind, Token_Kind.Ident) == .Ident {
|
||||
// e.g. <odin - Start of new element.
|
||||
_meas_begin(m, .elements)
|
||||
element = new_element(doc)
|
||||
if element == 0 { // First Element
|
||||
parent = element
|
||||
@@ -212,8 +279,14 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
|
||||
doc.elements[element].parent = parent
|
||||
doc.elements[element].ident = open.text
|
||||
_meas_end(m, .elements)
|
||||
|
||||
_meas_begin(m, .attributes)
|
||||
// PERF: Hot !!
|
||||
parse_attributes(doc, &doc.elements[element].attribs) or_return
|
||||
_meas_end(m, .attributes)
|
||||
|
||||
_meas_begin(m, .elements)
|
||||
|
||||
// If a DOCTYPE is present _or_ the caller
|
||||
// asked for a specific DOCTYPE and the DOCTYPE
|
||||
@@ -245,8 +318,11 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
return
|
||||
}
|
||||
|
||||
_meas_end(m, .elements)
|
||||
|
||||
} else if open.kind == .Slash {
|
||||
// Close tag.
|
||||
_meas_begin(m, .elements)
|
||||
ident := expect(t, .Ident) or_return
|
||||
_ = expect(t, .Gt) or_return
|
||||
|
||||
@@ -256,6 +332,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
}
|
||||
parent = doc.elements[element].parent
|
||||
element = parent
|
||||
_meas_end(m, .elements)
|
||||
|
||||
} else if open.kind == .Exclaim {
|
||||
// <!
|
||||
@@ -290,6 +367,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
// Comment: <!-- -->.
|
||||
// The grammar does not allow a comment to end in --->
|
||||
expect(t, .Dash)
|
||||
_meas_begin(m, .comments)
|
||||
comment := scan_comment(t) or_return
|
||||
|
||||
if .Intern_Comments in opts.flags {
|
||||
@@ -303,6 +381,7 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
append(&doc.elements[element].value, el)
|
||||
}
|
||||
}
|
||||
_meas_end(m, .comments)
|
||||
|
||||
case .Open_Bracket:
|
||||
// This could be a CDATA tag part of a tag's body. Unread the `<![`
|
||||
@@ -312,7 +391,9 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
// Instead of calling `parse_body` here, we could also `continue loop`
|
||||
// and fall through to the `case:` at the bottom of the outer loop.
|
||||
// This makes the intent clearer.
|
||||
_meas_begin(m, .cdata)
|
||||
parse_body(doc, element, opts) or_return
|
||||
_meas_end(m, .cdata)
|
||||
|
||||
case:
|
||||
error(t, t.offset, "Unexpected Token after <!: %#v", next)
|
||||
@@ -348,7 +429,9 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
|
||||
case:
|
||||
// This should be a tag's body text.
|
||||
_meas_begin(m, .text_content)
|
||||
parse_body(doc, element, opts) or_return
|
||||
_meas_end(m, .text_content)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -360,7 +443,9 @@ parse_bytes :: proc(data: []u8, options := DEFAULT_OPTIONS, path := "", error_ha
|
||||
return doc, .No_DocType
|
||||
}
|
||||
|
||||
_meas_begin(m, .resize_result)
|
||||
resize(&doc.elements, int(doc.element_count))
|
||||
_meas_end(m, .resize_result)
|
||||
return doc, .None
|
||||
}
|
||||
|
||||
@@ -408,7 +493,7 @@ validate_options :: proc(options: Options) -> (validated: Options, err: Error) {
|
||||
return validated, .None
|
||||
}
|
||||
|
||||
expect :: proc(t: ^Tokenizer, kind: Token_Kind, multiline_string := false) -> (tok: Token, err: Error) {
|
||||
expect :: #force_inline proc(t: ^Tokenizer, kind: Token_Kind, multiline_string := false) -> (tok: Token, err: Error) {
|
||||
tok = scan(t, multiline_string=multiline_string)
|
||||
if tok.kind == kind { return tok, .None }
|
||||
|
||||
@@ -416,18 +501,34 @@ expect :: proc(t: ^Tokenizer, kind: Token_Kind, multiline_string := false) -> (t
|
||||
return tok, .Unexpected_Token
|
||||
}
|
||||
|
||||
parse_attribute :: proc(doc: ^Document) -> (attr: Attribute, offset: int, err: Error) {
|
||||
_parse_attribute :: proc(doc: ^Document, builder: ^strings.Builder) -> (attr: Attribute, offset: int, err: Error) {
|
||||
assert(doc != nil)
|
||||
m := &_parse_bytes_meas
|
||||
_meas_begin(m, .parse_attribute_set_context)
|
||||
context.allocator = doc.allocator
|
||||
_meas_end(m, .parse_attribute_set_context)
|
||||
t := doc.tokenizer
|
||||
|
||||
_meas_begin(m, .parse_attribute_tokenize_ident)
|
||||
key := expect(t, .Ident) or_return
|
||||
_ = expect(t, .Eq) or_return
|
||||
value := expect(t, .String, multiline_string=true) or_return
|
||||
_meas_end(m, .parse_attribute_tokenize_ident)
|
||||
|
||||
normalized, normalize_err := entity.decode_xml(value.text, {.Normalize_Whitespace}, doc.allocator)
|
||||
_meas_begin(m, .parse_attribute_tokenize_eq)
|
||||
_ = expect(t, .Eq) or_return
|
||||
_meas_end(m, .parse_attribute_tokenize_eq)
|
||||
|
||||
_meas_begin(m, .parse_attribute_tokenize_value)
|
||||
value := expect(t, .String, multiline_string=true) or_return
|
||||
_meas_end(m, .parse_attribute_tokenize_value)
|
||||
|
||||
_meas_begin(m, .parse_attribute_decode)
|
||||
//builder := strings.builder_make_len_cap(0, len(value.text) + 10, doc.allocator)
|
||||
normalized, normalize_err := entity.decode_xml_sb(builder, value.text, {.Normalize_Whitespace}, doc.allocator)
|
||||
_meas_end(m, .parse_attribute_decode)
|
||||
if normalize_err == .None {
|
||||
append(&doc.strings_to_free, normalized)
|
||||
//_meas_begin(m, .parse_attribute_append)
|
||||
//append(&doc.strings_to_free, strings.to_string(builder))
|
||||
//_meas_end(m, .parse_attribute_append)
|
||||
value.text = normalized
|
||||
}
|
||||
|
||||
@@ -445,20 +546,56 @@ check_duplicate_attributes :: proc(t: ^Tokenizer, attribs: Attributes, attr: Att
|
||||
return .Duplicate_Attribute
|
||||
}
|
||||
}
|
||||
return .None
|
||||
return .None
|
||||
}
|
||||
|
||||
parse_attributes :: proc(doc: ^Document, attribs: ^Attributes) -> (err: Error) {
|
||||
assert(doc != nil)
|
||||
context.allocator = doc.allocator
|
||||
m := &_parse_bytes_meas
|
||||
_meas_begin(m, .attributes_set_context)
|
||||
context.allocator = doc.allocator // PERF: mildly warmish
|
||||
_meas_end(m, .attributes_set_context)
|
||||
t := doc.tokenizer
|
||||
|
||||
for peek(t).kind == .Ident {
|
||||
attr, offset := parse_attribute(doc) or_return
|
||||
check_duplicate_attributes(t, attribs^, attr, offset) or_return
|
||||
append(attribs, attr)
|
||||
// PERF: Hot path. Consolidate attribute strings into one buffer.
|
||||
// !!!!!!!!!!!!!!!!!
|
||||
// TODO: Cannot leave it like this !! This will cause strings to become invalid in case it ever needs to grow the buffer !!
|
||||
// !!!!!!!!!!!!!!!!!
|
||||
builder := strings.builder_make_len_cap(0, 500, doc.allocator)
|
||||
defer {
|
||||
append(&doc.strings_to_free, strings.to_string(builder))
|
||||
}
|
||||
|
||||
for {
|
||||
_meas_begin(m, .attributes_peek)
|
||||
k := peek(t).kind
|
||||
_meas_end(m, .attributes_peek)
|
||||
if k != .Ident {
|
||||
break
|
||||
}
|
||||
|
||||
_meas_begin(m, .attributes_parse_attribute)
|
||||
attr, offset := _parse_attribute(doc, &builder) or_return
|
||||
_meas_end(m, .attributes_parse_attribute)
|
||||
|
||||
_meas_begin(m, .attributes_check_duplicates)
|
||||
check_duplicate_attributes(t, attribs^, attr, offset) or_return
|
||||
_meas_end(m, .attributes_check_duplicates)
|
||||
|
||||
_meas_begin(m, .attributes_append)
|
||||
append(attribs, attr)
|
||||
_meas_end(m, .attributes_append)
|
||||
}
|
||||
|
||||
//log.infof("Builder contents: %v", strings.to_string(builder))
|
||||
//for attr in attribs {
|
||||
// log.infof("Attr: %v", attr.val)
|
||||
//}
|
||||
|
||||
_meas_begin(m, .attributes_skip_whitespace)
|
||||
skip_whitespace(t)
|
||||
_meas_end(m, .attributes_skip_whitespace)
|
||||
|
||||
return .None
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user