diff --git a/src/checker.cpp b/src/checker.cpp index fd65d42fc..01702b14e 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -7803,9 +7803,7 @@ gb_internal void check_parsed_files(Checker *c) { token.pos.column = 1; if (s->pkg->files.count > 0) { AstFile *f = s->pkg->files[0]; - if (f->tokens.count > 0) { - token = f->tokens[0]; - } + token = f->first_token; } error(token, "Undefined entry point procedure 'main'"); diff --git a/src/error.cpp b/src/error.cpp index 5879fbeec..f19829928 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -812,6 +812,10 @@ gb_internal void error_no_newline_va(TokenPos const &pos, char const *fmt, va_li gb_internal void syntax_error_va(TokenPos const &pos, TokenPos end, char const *fmt, va_list va) { + if (global_error_mute_depth > 0) { + global_error_mute_count += 1; + return; + } global_error_collector.count.fetch_add(1); mutex_lock(&global_error_collector.mutex); if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { @@ -844,6 +848,10 @@ gb_internal void syntax_error_va(TokenPos const &pos, TokenPos end, char const * } gb_internal void syntax_error_with_verbose_va(TokenPos const &pos, TokenPos end, char const *fmt, va_list va) { + if (global_error_mute_depth > 0) { + global_error_mute_count += 1; + return; + } global_error_collector.count.fetch_add(1); mutex_lock(&global_error_collector.mutex); if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { diff --git a/src/main.cpp b/src/main.cpp index 17306bfe1..b541dc0aa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2390,12 +2390,10 @@ gb_internal void show_timings(Checker *c, Timings *t) { isize files = 0; isize packages = p->packages.count; isize total_file_size = 0; - f64 total_tokenizing_time = 0; f64 total_parsing_time = 0; for (AstPackage *pkg : p->packages) { files += pkg->files.count; for (AstFile *file : pkg->files) { - total_tokenizing_time += file->time_to_tokenize; total_parsing_time += file->time_to_parse; total_file_size += file->tokenizer.end - file->tokenizer.start; } @@ -2419,22 +2417,9 @@ gb_internal void show_timings(Checker *c, Timings *t) { gb_printf_err("Total File Size - %td\n", total_file_size); gb_printf_err("\n"); } - { - f64 time = total_tokenizing_time; - gb_printf_err("Tokenization Only\n"); - gb_printf_err("LOC/s - %.3f\n", cast(f64)lines/time); - gb_printf_err("us/LOC - %.3f\n", 1.0e6*time/cast(f64)lines); - gb_printf_err("Tokens/s - %.3f\n", cast(f64)tokens/time); - gb_printf_err("us/Token - %.3f\n", 1.0e6*time/cast(f64)tokens); - gb_printf_err("bytes/s - %.3f\n", cast(f64)total_file_size/time); - gb_printf_err("MiB/s - %.3f\n", cast(f64)(total_file_size/time)/(1024*1024)); - gb_printf_err("us/bytes - %.3f\n", 1.0e6*time/cast(f64)total_file_size); - - gb_printf_err("\n"); - } { f64 time = total_parsing_time; - gb_printf_err("Parsing Only\n"); + gb_printf_err("Tokenizing and Parsing Only\n"); gb_printf_err("LOC/s - %.3f\n", cast(f64)lines/time); gb_printf_err("us/LOC - %.3f\n", 1.0e6*time/cast(f64)lines); gb_printf_err("Tokens/s - %.3f\n", cast(f64)tokens/time); @@ -3631,7 +3616,7 @@ gb_internal gbFileError write_file_with_stripped_tokens(gbFile *f, AstFile *file u8 const *file_data = file->tokenizer.start; i32 prev_offset = 0; i32 const end_offset = cast(i32)(file->tokenizer.end - file->tokenizer.start); - for (Token const &token : file->tokens) { + for (Token const &token : file->token_edits) { if (token.flags & (TokenFlag_Remove|TokenFlag_Replace)) { i32 offset = token.pos.offset; i32 to_write = offset-prev_offset; @@ -3674,15 +3659,7 @@ gb_internal int strip_semicolons(Parser *parser) { for (AstPackage *pkg : parser->packages) { for (AstFile *file : pkg->files) { - bool nothing_to_change = true; - for (Token const &token : file->tokens) { - if (token.flags) { - nothing_to_change = false; - break; - } - } - - if (nothing_to_change) { + if (file->token_edits.count == 0) { continue; } diff --git a/src/parser.cpp b/src/parser.cpp index 3f6578ab4..31561f39c 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1575,13 +1575,35 @@ gb_internal Ast *ast_attribute(AstFile *f, Token token, Token open, Token close, } -gb_internal bool next_token0(AstFile *f) { - if (f->curr_token_index+1 < f->tokens.count) { - f->curr_token = f->tokens[++f->curr_token_index]; - return true; +gb_internal Token tokenize_next(AstFile *f) { + Token token = {}; + tokenizer_get_token(&f->tokenizer, &token); + f->token_count += 1; + if (token.kind == Token_Invalid) { + f->invalid_token_pos = token.pos; + syntax_error(token.pos, "Failed to parse file: %.*s; invalid token found in file", LIT(f->fullpath)); + begin_error_mute(); + token.kind = Token_EOF; } - syntax_error(f->curr_token, "Token is EOF"); - return false; + return token; +} + +gb_internal bool next_token0(AstFile *f) { + if (f->curr_token.kind == Token_EOF) { + syntax_error(f->curr_token, "Token is EOF"); + return false; + } + f->curr_token_index += 1; + if (f->lookahead_index < f->lookahead.count) { + f->curr_token = f->lookahead[f->lookahead_index++]; + if (f->lookahead_index == f->lookahead.count) { + array_clear(&f->lookahead); + f->lookahead_index = 0; + } + } else { + f->curr_token = tokenize_next(f); + } + return true; } @@ -1665,7 +1687,6 @@ gb_internal Token advance_token(AstFile *f) { f->lead_comment = nullptr; f->line_comment = nullptr; - f->prev_token_index = f->curr_token_index; Token prev = f->prev_token = f->curr_token; bool ok = next_token0(f); @@ -1688,30 +1709,40 @@ gb_internal Token advance_token(AstFile *f) { } -gb_internal Token peek_token(AstFile *f) { - for (isize i = f->curr_token_index+1; i < f->tokens.count; i++) { - Token tok = f->tokens[i]; +gb_internal Token peek_token_n(AstFile *f, isize n) { + if (f->curr_token.kind == Token_EOF) { + return {}; + } + for (isize i = f->lookahead_index; /**/; i++) { + if (i == f->lookahead.count) { + array_add(&f->lookahead, tokenize_next(f)); + } + Token tok = f->lookahead[i]; if (tok.kind == Token_Comment) { continue; } - return tok; + if (n-- == 0) { + return tok; + } + if (tok.kind == Token_EOF) { + return {}; + } } - return {}; } -gb_internal Token peek_token_n(AstFile *f, isize n) { - Token found = {}; - for (isize i = f->curr_token_index+1; i < f->tokens.count; i++) { - Token tok = f->tokens[i]; - if (tok.kind == Token_Comment) { - continue; - } - found = tok; - if (n-- == 0) { - return found; - } +gb_internal Token peek_token(AstFile *f) { + return peek_token_n(f, 0); +} + +// the next token, even if it is a comment +gb_internal Token peek_raw_token(AstFile *f) { + if (f->curr_token.kind == Token_EOF) { + return f->curr_token; } - return {}; + if (f->lookahead_index == f->lookahead.count) { + array_add(&f->lookahead, tokenize_next(f)); + } + return f->lookahead[f->lookahead_index]; } @@ -1778,6 +1809,9 @@ gb_internal Token expect_token(AstFile *f, TokenKind kind) { end_error_block(); if (prev.kind == Token_EOF) { + if (f->invalid_token_pos.line != 0) { + end_error_mute(); + } exit_with_errors(); } } @@ -1827,6 +1861,18 @@ gb_internal bool is_token_range(Token tok) { } +gb_internal void add_token_edit(AstFile *f, Token token, u8 flag) { + if (f->token_edits.count > 0) { + Token *last = &f->token_edits[f->token_edits.count-1]; + if (last->pos.offset == token.pos.offset) { + last->flags |= flag; + return; + } + } + token.flags |= flag; + array_add(&f->token_edits, token); +} + gb_internal Token expect_operator(AstFile *f) { Token prev = f->curr_token; if ((prev.kind == Token_in || prev.kind == Token_not_in) && (f->expr_level >= 0 || f->allow_in_expr)) { @@ -1847,7 +1893,7 @@ gb_internal Token expect_operator(AstFile *f) { } if (prev.kind == Token_Ellipsis) { syntax_error(prev, "'..' for ranges are not allowed, did you mean '..<' or '..='?"); - f->tokens[f->curr_token_index].flags |= TokenFlag_Replace; + add_token_edit(f, prev, TokenFlag_Replace); } advance_token(f); @@ -1962,8 +2008,8 @@ gb_internal Token expect_closing(AstFile *f, TokenKind kind, String const &conte gb_internal void assign_removal_flag_to_semicolon(AstFile *f) { // NOTE(bill): this is used for rewriting files to strip unneeded semicolons - Token *prev_token = &f->tokens[f->prev_token_index]; - Token *curr_token = &f->tokens[f->curr_token_index]; + Token const *prev_token = &f->prev_token; + Token const *curr_token = &f->curr_token; GB_ASSERT(prev_token->kind == Token_Semicolon); if (prev_token->string != ";") { return; @@ -1987,7 +2033,7 @@ gb_internal void assign_removal_flag_to_semicolon(AstFile *f) { if (is_strict_style(f) || (ast_file_vet_flags(f) & VetFlag_Semicolon)) { syntax_error(*prev_token, "Found unneeded semicolon"); } - prev_token->flags |= TokenFlag_Remove; + add_token_edit(f, *prev_token, TokenFlag_Remove); } gb_internal void expect_semicolon(AstFile *f) { @@ -2262,11 +2308,7 @@ gb_internal Ast *convert_stmt_to_expr(AstFile *f, Ast *statement, String const & } syntax_error(f->curr_token, "Expected '%.*s', found a simple statement.", LIT(kind)); - Token end = f->curr_token; - if (f->tokens.count < f->curr_token_index) { - end = f->tokens[f->curr_token_index+1]; - } - return ast_bad_expr(f, f->curr_token, end); + return ast_bad_expr(f, f->curr_token, f->curr_token); } gb_internal Ast *convert_stmt_to_body(AstFile *f, Ast *stmt) { @@ -5471,7 +5513,7 @@ if_else_chain:; break; default: syntax_error(f->curr_token, "Expected if statement block statement"); - else_stmt = ast_bad_stmt(f, f->curr_token, f->tokens[f->curr_token_index+1]); + else_stmt = ast_bad_stmt(f, f->curr_token, peek_raw_token(f)); break; } } @@ -5529,7 +5571,7 @@ gb_internal Ast *parse_when_stmt(AstFile *f) { } break; default: syntax_error(f->curr_token, "Expected when statement block statement"); - else_stmt = ast_bad_stmt(f, f->curr_token, f->tokens[f->curr_token_index+1]); + else_stmt = ast_bad_stmt(f, f->curr_token, peek_raw_token(f)); break; } } @@ -6357,7 +6399,7 @@ gb_internal Array parse_stmt_list(AstFile *f) { } -gb_internal ParseFileError init_ast_file(AstFile *f, String const &fullpath, TokenPos *err_pos) { +gb_internal ParseFileError init_ast_file(AstFile *f, String const &fullpath) { GB_ASSERT(f != nullptr); f->fullpath = string_trim_whitespace(fullpath); // Just in case f->filename = remove_directory_from_path(f->fullpath); @@ -6387,51 +6429,23 @@ gb_internal ParseFileError init_ast_file(AstFile *f, String const &fullpath, Tok } - isize file_size = f->tokenizer.end - f->tokenizer.start; - - // NOTE(bill): Determine allocation size required for tokens - isize token_cap = file_size/3ll; - isize pow2_cap = gb_max(cast(isize)prev_pow2(cast(i64)token_cap)/2, 16); - token_cap = ((token_cap + pow2_cap-1)/pow2_cap) * pow2_cap; - - isize init_token_cap = gb_max(token_cap, 16); - array_init(&f->tokens, ast_allocator(f), 0, gb_max(init_token_cap, 16)); + array_init(&f->lookahead, ast_allocator(f), 0, 4); + array_init(&f->token_edits, ast_allocator(f), 0, 0); + array_init(&f->comments, ast_allocator(f), 0, 0); + array_init(&f->imports, ast_allocator(f), 0, 0); if (err == TokenizerInit_Empty) { Token token = {Token_EOF}; token.pos.file_id = f->id; token.pos.line = 1; token.pos.column = 1; - array_add(&f->tokens, token); + f->token_count = 1; + f->first_token = f->prev_token = f->curr_token = token; return ParseFile_None; } - u64 start = time_stamp_time_now(); - - for (;;) { - Token *token = array_add_and_get(&f->tokens); - tokenizer_get_token(&f->tokenizer, token); - if (token->kind == Token_Invalid) { - err_pos->line = token->pos.line; - err_pos->column = token->pos.column; - return ParseFile_InvalidToken; - } - - if (token->kind == Token_EOF) { - break; - } - } - - u64 end = time_stamp_time_now(); - f->time_to_tokenize = cast(f64)(end-start)/cast(f64)time_stamp__freq(); - - f->prev_token_index = 0; f->curr_token_index = 0; - f->prev_token = f->tokens[f->prev_token_index]; - f->curr_token = f->tokens[f->curr_token_index]; - - array_init(&f->comments, ast_allocator(f), 0, 0); - array_init(&f->imports, ast_allocator(f), 0, 0); + f->first_token = f->prev_token = f->curr_token = tokenize_next(f); f->curr_proc = nullptr; @@ -6440,7 +6454,8 @@ gb_internal ParseFileError init_ast_file(AstFile *f, String const &fullpath, Tok gb_internal void destroy_ast_file(AstFile *f) { GB_ASSERT(f != nullptr); - array_free(&f->tokens); + array_free(&f->lookahead); + array_free(&f->token_edits); array_free(&f->comments); array_free(&f->imports); } @@ -7473,11 +7488,8 @@ gb_internal bool parse_file_tag(const String &lc, const Token &tok, AstFile *f) } gb_internal bool parse_file(Parser *p, AstFile *f) { - if (f->tokens.count == 0) { - return true; - } - if (f->tokens.count > 0 && f->tokens[0].kind == Token_EOF) { - return true; + if (f->first_token.kind == Token_EOF) { + return f->invalid_token_pos.line == 0; } u64 start = time_stamp_time_now(); @@ -7604,9 +7616,7 @@ gb_internal ParseFileError process_imported_file(Parser *p, ImportedFile importe AstFile *file = permanent_alloc_item(); file->pkg = pkg; file->id = cast(i32)(imported_file.index+1); - TokenPos err_pos = {0}; - ParseFileError err = init_ast_file(file, fi.fullpath, &err_pos); - err_pos.file_id = file->id; + ParseFileError err = init_ast_file(file, fi.fullpath); file->last_error = err; if (err != ParseFile_None) { @@ -7629,9 +7639,6 @@ gb_internal ParseFileError process_imported_file(Parser *p, ImportedFile importe case ParseFile_NotFound: syntax_error(pos, "Failed to parse file: %.*s; file cannot be found ('%.*s')", LIT(fi.name), LIT(fi.fullpath)); break; - case ParseFile_InvalidToken: - syntax_error(err_pos, "Failed to parse file: %.*s; invalid token found in file", LIT(fi.name)); - break; case ParseFile_EmptyFile: syntax_error(pos, "Failed to parse file: %.*s; file contains no tokens", LIT(fi.name)); break; @@ -7660,7 +7667,14 @@ gb_internal ParseFileError process_imported_file(Parser *p, ImportedFile importe } - if (parse_file(p, file)) { + bool parsed = parse_file(p, file); + if (file->invalid_token_pos.line != 0) { + end_error_mute(); + file->last_error = ParseFile_InvalidToken; + return ParseFile_InvalidToken; + } + + if (parsed) { MUTEX_GUARD_BLOCK(&pkg->files_mutex) { array_add(&pkg->files, file); } @@ -7669,7 +7683,7 @@ gb_internal ParseFileError process_imported_file(Parser *p, ImportedFile importe if (pkg->name.len == 0) { pkg->name = file->package_name; } else if (pkg->name != file->package_name) { - if (file->tokens.count > 0 && file->tokens[0].kind != Token_EOF) { + if (file->first_token.kind != Token_EOF) { Token tok = file->package_token; tok.pos.file_id = file->id; tok.pos.line = gb_max(tok.pos.line, 1); @@ -7683,7 +7697,7 @@ gb_internal ParseFileError process_imported_file(Parser *p, ImportedFile importe mutex_unlock(&pkg->name_mutex); p->total_line_count.fetch_add(file->tokenizer.line_count); - p->total_token_count.fetch_add(file->tokens.count); + p->total_token_count.fetch_add(file->token_count); } return ParseFile_None; diff --git a/src/parser.hpp b/src/parser.hpp index d1988f89b..1f5869883 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -118,11 +118,15 @@ struct AstFile { String directory; Tokenizer tokenizer; - Array tokens; + Array lookahead; // read by peeking, before the parser reaches them + isize lookahead_index; + isize token_count; isize curr_token_index; - isize prev_token_index; + Token first_token; Token curr_token; Token prev_token; // previous non-comment + TokenPos invalid_token_pos; + Array token_edits; // for `-strip-semicolon` Token package_token; String package_name; @@ -151,8 +155,7 @@ struct AstFile { Ast * curr_proc; isize error_count; ParseFileError last_error; - f64 time_to_tokenize; // seconds - f64 time_to_parse; // seconds + f64 time_to_parse; // seconds, including tokenizing CommentGroup *lead_comment; // Comment (block) before the decl CommentGroup *line_comment; // Comment after the semicolon