diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index d3bf23519..6900a2098 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -154,16 +154,29 @@ GB_STATIC_ASSERT(Token__KeywordEnd-Token__KeywordBegin <= gb_count_of(keyword_ha gb_global isize const min_keyword_size = 2; gb_global isize max_keyword_size = 11; gb_global bool keyword_indices[16] = {}; +gb_global u32 keyword_first_letters = 0; // a bit for each letter from 'a' which a keyword starts with gb_internal gb_inline u32 keyword_hash(u8 const *text, isize len) { - return fnv32a(text, len); + return cast(u32)len + text[0] + 3*text[1] + 26*text[len-1]; } + +gb_internal gb_inline bool could_be_keyword(String const &s) { + if (s.len < min_keyword_size || s.len > max_keyword_size || !keyword_indices[s.len]) { + return false; + } + u32 first = cast(u32)s[0] - 'a'; + return first < 26 && (keyword_first_letters & (1u< t->read_curr) { + t->column_minus_one += cast(i32)(p - t->read_curr); + t->curr = p-1; + t->curr_rune = p[-1]; + t->read_curr = p; + } +} + gb_internal void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void const *data, isize size) { t->fullpath = fullpath; t->column_minus_one = -1; @@ -663,10 +686,24 @@ gb_internal bool scan_escape(Tokenizer *t) { gb_internal gb_inline void tokenizer_skip_line(Tokenizer *t) { while (t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) { + u8 *p = t->read_curr; + while (p < t->end && *p != '\n' && *p != 0 && *p < 0x80) { + p++; + } + tokenizer_skip_ascii_to(t, p); advance_to_next_rune(t); } } +gb_internal gb_inline void tokenizer_skip_spaces(Tokenizer *t) { + u8 *p = t->read_curr; + while (p < t->end && (*p == ' ' || *p == '\t' || *p == '\r')) { + p++; + } + tokenizer_skip_ascii_to(t, p); + advance_to_next_rune(t); +} + gb_internal gb_inline void tokenizer_skip_whitespace(Tokenizer *t, bool on_newline) { if (on_newline) { for (;;) { @@ -674,7 +711,7 @@ gb_internal gb_inline void tokenizer_skip_whitespace(Tokenizer *t, bool on_newli case ' ': case '\t': case '\r': - advance_to_next_rune(t); + tokenizer_skip_spaces(t); continue; } break; @@ -683,10 +720,12 @@ gb_internal gb_inline void tokenizer_skip_whitespace(Tokenizer *t, bool on_newli for (;;) { switch (t->curr_rune) { case '\n': + advance_to_next_rune(t); + continue; case ' ': case '\t': case '\r': - advance_to_next_rune(t); + tokenizer_skip_spaces(t); continue; } break; @@ -710,6 +749,11 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { Rune curr_rune = t->curr_rune; if (rune_is_letter(curr_rune)) { token->kind = Token_Ident; + u8 *p = t->read_curr; + while (p < t->end && *p < 0x80 && rune_is_letter_or_digit(*p)) { + p++; + } + tokenizer_skip_ascii_to(t, p); while (rune_is_letter_or_digit(t->curr_rune)) { advance_to_next_rune(t); } @@ -717,7 +761,7 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { token->string.len = t->curr - token->string.text; // NOTE(bill): Heavily optimize to make it faster to find keywords - if (1 < token->string.len && token->string.len <= max_keyword_size && keyword_indices[token->string.len]) { + if (could_be_keyword(token->string)) { u32 hash = keyword_hash(token->string.text, token->string.len); u32 index = hash & KEYWORD_HASH_TABLE_MASK; KeywordHashEntry *entry = &keyword_hash_table[index];