diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index a9a067ae0..eb93b7919 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -203,6 +203,7 @@ struct lbGenerator : LinkerData { PtrMap modules; // key is `AstPackage *` (`void *` is used for future use) PtrMap modules_through_ctx; + PtrMap file_modules; // NOTE(bill): of the files of a package which is split into several modules lbModule default_module; lbModule *equal_module; diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index dc3d51f8e..8d6208536 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -185,6 +185,87 @@ gb_internal void lb_init_module(lbModule *m, bool do_threading) { } } +// NOTE(bill): an estimate of the code generated for each file: the size of the bodies of its procedures which are used, +// polymorphic instances included as each is generated with its procedure's file +gb_internal i64 lb_estimate_file_code_sizes(CheckerInfo *info, PtrMap *sizes) { + i64 total = 0; + for (Entity *e : info->entities) { + if (e->kind != Entity_Procedure || e->file == nullptr || e->min_dep_count.load(std::memory_order_relaxed) == 0) { + continue; + } + if (e->scope == nullptr || (e->scope->flags & ScopeFlag_File) == 0) { + continue; + } + DeclInfo *d = e->decl_info; + if (d == nullptr || d->proc_lit == nullptr || d->proc_lit->kind != Ast_ProcLit || d->proc_lit->ProcLit.body == nullptr) { + continue; + } + Ast *body = d->proc_lit->ProcLit.body; + i64 size = gb_max(cast(i64)(ast_end_token(body).pos.offset - ast_token(body).pos.offset), 1); + i64 *found = map_get(sizes, e->file); + if (found) { + *found += size; + } else { + map_set(sizes, e->file, size); + } + total += size; + } + return total; +} + +gb_internal void lb_split_package_into_modules(lbGenerator *gen, Checker *c, AstPackage *pkg, PtrMap *file_sizes, i64 share, bool do_threading) { + i64 pkg_size = 0; + for (AstFile *f : pkg->files) { + i64 *size = map_get(file_sizes, f); + pkg_size += size ? *size : 0; + } + if (pkg_size <= share + share/2) { + return; + } + + auto files = array_make(heap_allocator(), 0, pkg->files.count); + defer (array_free(&files)); + for (AstFile *f : pkg->files) { + array_add(&files, f); + } + array_sort(files, [](void const *a, void const *b) -> int { + return string_compare((*cast(AstFile **)a)->fullpath, (*cast(AstFile **)b)->fullpath); + }); + + i64 group_count = (pkg_size + share - 1) / share; + i64 group_size_target = pkg_size / group_count; + + lbModule *group = nullptr; + i64 group_size = 0; + for (AstFile *f : files) { + if (group == nullptr || group_size >= group_size_target) { + group = permanent_alloc_item(); + group->file = f; + group->pkg = pkg; + group->gen = gen; + group->checker = c; + map_set(&gen->modules, cast(void *)group, group); // point to itself just add it to the list + lb_init_module(group, do_threading); + + if (LLVM_WEAK_MONOMORPHIZATION) { + auto pm = permanent_alloc_item(); + pm->file = f; + pm->pkg = pkg; + pm->gen = gen; + pm->checker = c; + group->polymorphic_module = pm; + pm->polymorphic_module = pm; + map_set(&gen->modules, cast(void *)pm, pm); + lb_init_module(pm, do_threading); + } + group_size = 0; + } + map_set(&gen->file_modules, f, group); + i64 *size = map_get(file_sizes, f); + group_size += size ? *size : 0; + } +} + gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) { if (global_error_collector.count != 0) { return false; @@ -206,9 +287,17 @@ gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) { map_init(&gen->modules, gen->info->packages.count*2); map_init(&gen->modules_through_ctx, gen->info->packages.count*2); + map_init(&gen->file_modules); if (USE_SEPARATE_MODULES) { bool module_per_file = build_context.module_per_file && (build_context.optimization_level <= 0 || build_context.lto_kind != LTO_None); + // NOTE(bill): A package with much more code than an even share of all of it is generated in several modules + // of contiguous files, as each module is generated by one thread. The share does not depend on the number of + // threads, so neither do the modules. + PtrMap file_sizes = {}; + map_init(&file_sizes, 1024); + i64 share = gb_max(lb_estimate_file_code_sizes(gen->info, &file_sizes) / 32, 1); + for (auto const &entry : gen->info->packages) { AstPackage *pkg = entry.value; auto m = permanent_alloc_item(); @@ -232,6 +321,9 @@ gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) { } bool allow_for_per_file = pkg->kind == Package_Runtime || module_per_file; + if (!allow_for_per_file) { + lb_split_package_into_modules(gen, c, pkg, &file_sizes, share, do_threading); + } #if 0 if (!allow_for_per_file) { @@ -572,6 +664,10 @@ gb_internal lbModule *lb_module_of_expr(lbGenerator *gen, Ast *expr) { if (found) { return *found; } + found = map_get(&gen->file_modules, file); + if (found) { + return *found; + } if (file->pkg) { found = map_get(&gen->modules, cast(void *)file->pkg); @@ -599,6 +695,11 @@ gb_internal lbModule *lb_module_of_entity_internal(lbGenerator *gen, Entity *e, GB_ASSERT(*found != nullptr); return *found; } + found = map_get(&gen->file_modules, e->file); + if (found) { + GB_ASSERT(*found != nullptr); + return *found; + } } if (e->pkg) { found = map_get(&gen->modules, cast(void *)e->pkg);