mirror/Odin
1
0 Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-10-09 06:12:05 -04:00
Code Issues Packages Projects Releases Wiki Activity

Tokenize whilst parsing rather than storing every token of a file first

Browse Source
This commit is contained in:
gingerBill committed 2026-10-03 16:03:59 +01:00
1 parent fcab1c1887
commit 32f30c3eb3
5 files changed
+119 -119
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/main.cpp
+3 -26
View File
@@ -2390,12 +2390,10 @@ gb_internal void show_timings(Checker *c, Timings *t) {
isize files = 0;
isize packages = p->packages.count;
isize total_file_size = 0;
f64 total_tokenizing_time = 0;
f64 total_parsing_time = 0;
for (AstPackage *pkg : p->packages) {
files += pkg->files.count;
for (AstFile *file : pkg->files) {
total_tokenizing_time += file->time_to_tokenize;
total_parsing_time += file->time_to_parse;
total_file_size += file->tokenizer.end - file->tokenizer.start;
}
@@ -2419,22 +2417,9 @@ gb_internal void show_timings(Checker *c, Timings *t) {
gb_printf_err("Total File Size - %td\n", total_file_size);
gb_printf_err("\n");
}
{
f64 time = total_tokenizing_time;
gb_printf_err("Tokenization Only\n");
gb_printf_err("LOC/s - %.3f\n", cast(f64)lines/time);
gb_printf_err("us/LOC - %.3f\n", 1.0e6*time/cast(f64)lines);
gb_printf_err("Tokens/s - %.3f\n", cast(f64)tokens/time);
gb_printf_err("us/Token - %.3f\n", 1.0e6*time/cast(f64)tokens);
gb_printf_err("bytes/s - %.3f\n", cast(f64)total_file_size/time);
gb_printf_err("MiB/s - %.3f\n", cast(f64)(total_file_size/time)/(1024*1024));
gb_printf_err("us/bytes - %.3f\n", 1.0e6*time/cast(f64)total_file_size);
gb_printf_err("\n");
}
{
f64 time = total_parsing_time;
gb_printf_err("Parsing Only\n");
gb_printf_err("Tokenizing and Parsing Only\n");
gb_printf_err("LOC/s - %.3f\n", cast(f64)lines/time);
gb_printf_err("us/LOC - %.3f\n", 1.0e6*time/cast(f64)lines);
gb_printf_err("Tokens/s - %.3f\n", cast(f64)tokens/time);
@@ -3631,7 +3616,7 @@ gb_internal gbFileError write_file_with_stripped_tokens(gbFile *f, AstFile *file
u8 const *file_data = file->tokenizer.start;
i32 prev_offset = 0;
i32 const end_offset = cast(i32)(file->tokenizer.end - file->tokenizer.start);
for (Token const &token : file->tokens) {
for (Token const &token : file->token_edits) {
if (token.flags & (TokenFlag_Remove|TokenFlag_Replace)) {
i32 offset = token.pos.offset;
i32 to_write = offset-prev_offset;
@@ -3674,15 +3659,7 @@ gb_internal int strip_semicolons(Parser *parser) {
for (AstPackage *pkg : parser->packages) {
for (AstFile *file : pkg->files) {
bool nothing_to_change = true;
for (Token const &token : file->tokens) {
if (token.flags) {
nothing_to_change = false;
break;
}
}
if (nothing_to_change) {
if (file->token_edits.count == 0) {
continue;
}