Scan the runs of ASCII characters is identifiers, and simplify the hash for the keyword look up

src/tokenizer.cpp

@@ -154,16 +154,29 @@ GB_STATIC_ASSERT(Token__KeywordEnd-Token__KeywordBegin <= gb_count_of(keyword_ha
 gb_global isize const min_keyword_size = 2;
 gb_global isize max_keyword_size = 11;
 gb_global bool keyword_indices[16] = {};
+gb_global u32  keyword_first_letters = 0; // a bit for each letter from 'a' which a keyword starts with
 
 
 gb_internal gb_inline u32 keyword_hash(u8 const *text, isize len) {
-	return fnv32a(text, len);
+	return cast(u32)len + text[0] + 3*text[1] + 26*text[len-1];
 }
+
+gb_internal gb_inline bool could_be_keyword(String const &s) {
+	if (s.len < min_keyword_size || s.len > max_keyword_size || !keyword_indices[s.len]) {
+		return false;
+	}
+	u32 first = cast(u32)s[0] - 'a';
+	return first < 26 && (keyword_first_letters & (1u<<first)) != 0;
+}
+
 gb_internal void add_keyword_hash_entry(String const &s, TokenKind kind) {
 	max_keyword_size = gb_max(max_keyword_size, s.len);
 
 	keyword_indices[s.len] = true;
 
+	GB_ASSERT('a' <= s[0] && s[0] <= 'z');
+	keyword_first_letters |= 1u<<(s[0]-'a');
+
 	u32 hash = keyword_hash(s.text, s.len);
 
 	// NOTE(bill): This is a bit of an empirical hack in order to speed things up
@@ -390,6 +403,16 @@ gb_internal gb_inline void advance_to_next_rune(Tokenizer *t) {
 	advance_to_next_rune_slow(t);
 }
 
+// Skips over the characters before `p`, which are ASCII and not newlines, as `advance_to_next_rune` would one at a time
+gb_internal gb_inline void tokenizer_skip_ascii_to(Tokenizer *t, u8 *p) {
+	if (p > t->read_curr) {
+		t->column_minus_one += cast(i32)(p - t->read_curr);
+		t->curr      = p-1;
+		t->curr_rune = p[-1];
+		t->read_curr = p;
+	}
+}
+
 gb_internal void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void const *data, isize size) {
 	t->fullpath = fullpath;
 	t->column_minus_one = -1;
@@ -663,10 +686,24 @@ gb_internal bool scan_escape(Tokenizer *t) {
 
 gb_internal gb_inline void tokenizer_skip_line(Tokenizer *t) {
 	while (t->curr_rune != '\n' && t->curr_rune != GB_RUNE_EOF) {
+		u8 *p = t->read_curr;
+		while (p < t->end && *p != '\n' && *p != 0 && *p < 0x80) {
+			p++;
+		}
+		tokenizer_skip_ascii_to(t, p);
 		advance_to_next_rune(t);
 	}
 }
 
+gb_internal gb_inline void tokenizer_skip_spaces(Tokenizer *t) {
+	u8 *p = t->read_curr;
+	while (p < t->end && (*p == ' ' || *p == '\t' || *p == '\r')) {
+		p++;
+	}
+	tokenizer_skip_ascii_to(t, p);
+	advance_to_next_rune(t);
+}
+
 gb_internal gb_inline void tokenizer_skip_whitespace(Tokenizer *t, bool on_newline) {
 	if (on_newline) {
 		for (;;) {
@@ -674,7 +711,7 @@ gb_internal gb_inline void tokenizer_skip_whitespace(Tokenizer *t, bool on_newli
 			case ' ':
 			case '\t':
 			case '\r':
-				advance_to_next_rune(t);
+				tokenizer_skip_spaces(t);
 				continue;
 			}
 			break;
@@ -683,10 +720,12 @@ gb_internal gb_inline void tokenizer_skip_whitespace(Tokenizer *t, bool on_newli
 		for (;;) {
 			switch (t->curr_rune) {
 			case '\n':
+				advance_to_next_rune(t);
+				continue;
 			case ' ':
 			case '\t':
 			case '\r':
-				advance_to_next_rune(t);
+				tokenizer_skip_spaces(t);
 				continue;
 			}
 			break;
@@ -710,6 +749,11 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
 	Rune curr_rune = t->curr_rune;
 	if (rune_is_letter(curr_rune)) {
 		token->kind = Token_Ident;
+		u8 *p = t->read_curr;
+		while (p < t->end && *p < 0x80 && rune_is_letter_or_digit(*p)) {
+			p++;
+		}
+		tokenizer_skip_ascii_to(t, p);
 		while (rune_is_letter_or_digit(t->curr_rune)) {
 			advance_to_next_rune(t);
 		}
@@ -717,7 +761,7 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
 		token->string.len = t->curr - token->string.text;
 
 		// NOTE(bill): Heavily optimize to make it faster to find keywords
-		if (1 < token->string.len && token->string.len <= max_keyword_size && keyword_indices[token->string.len]) {
+		if (could_be_keyword(token->string)) {
 			u32 hash = keyword_hash(token->string.text, token->string.len);
 			u32 index = hash & KEYWORD_HASH_TABLE_MASK;
 			KeywordHashEntry *entry = &keyword_hash_table[index];