Generate a package with much more code than a 32nd of all of it in several modules of contiguous files, estimated by the bodies of its used procedures, as each module is generated by one thread

src/llvm_backend.hpp

@@ -203,6 +203,7 @@ struct lbGenerator : LinkerData {
 
 	PtrMap<void *, lbModule *> modules; // key is `AstPackage *` (`void *` is used for future use)
 	PtrMap<LLVMContextRef, lbModule *> modules_through_ctx; 
+	PtrMap<AstFile *, lbModule *> file_modules; // NOTE(bill): of the files of a package which is split into several modules
 	lbModule default_module;
 
 	lbModule *equal_module;

src/llvm_backend_general.cpp

@@ -185,6 +185,87 @@ gb_internal void lb_init_module(lbModule *m, bool do_threading) {
 	}
 }
 
+// NOTE(bill): an estimate of the code generated for each file: the size of the bodies of its procedures which are used,
+// polymorphic instances included as each is generated with its procedure's file
+gb_internal i64 lb_estimate_file_code_sizes(CheckerInfo *info, PtrMap<AstFile *, i64> *sizes) {
+	i64 total = 0;
+	for (Entity *e : info->entities) {
+		if (e->kind != Entity_Procedure || e->file == nullptr || e->min_dep_count.load(std::memory_order_relaxed) == 0) {
+			continue;
+		}
+		if (e->scope == nullptr || (e->scope->flags & ScopeFlag_File) == 0) {
+			continue;
+		}
+		DeclInfo *d = e->decl_info;
+		if (d == nullptr || d->proc_lit == nullptr || d->proc_lit->kind != Ast_ProcLit || d->proc_lit->ProcLit.body == nullptr) {
+			continue;
+		}
+		Ast *body = d->proc_lit->ProcLit.body;
+		i64 size = gb_max(cast(i64)(ast_end_token(body).pos.offset - ast_token(body).pos.offset), 1);
+		i64 *found = map_get(sizes, e->file);
+		if (found) {
+			*found += size;
+		} else {
+			map_set(sizes, e->file, size);
+		}
+		total += size;
+	}
+	return total;
+}
+
+gb_internal void lb_split_package_into_modules(lbGenerator *gen, Checker *c, AstPackage *pkg, PtrMap<AstFile *, i64> *file_sizes, i64 share, bool do_threading) {
+	i64 pkg_size = 0;
+	for (AstFile *f : pkg->files) {
+		i64 *size = map_get(file_sizes, f);
+		pkg_size += size ? *size : 0;
+	}
+	if (pkg_size <= share + share/2) {
+		return;
+	}
+
+	auto files = array_make<AstFile *>(heap_allocator(), 0, pkg->files.count);
+	defer (array_free(&files));
+	for (AstFile *f : pkg->files) {
+		array_add(&files, f);
+	}
+	array_sort(files, [](void const *a, void const *b) -> int {
+		return string_compare((*cast(AstFile **)a)->fullpath, (*cast(AstFile **)b)->fullpath);
+	});
+
+	i64 group_count = (pkg_size + share - 1) / share;
+	i64 group_size_target = pkg_size / group_count;
+
+	lbModule *group = nullptr;
+	i64 group_size = 0;
+	for (AstFile *f : files) {
+		if (group == nullptr || group_size >= group_size_target) {
+			group = permanent_alloc_item<lbModule>();
+			group->file    = f;
+			group->pkg     = pkg;
+			group->gen     = gen;
+			group->checker = c;
+			map_set(&gen->modules, cast(void *)group, group); // point to itself just add it to the list
+			lb_init_module(group, do_threading);
+
+			if (LLVM_WEAK_MONOMORPHIZATION) {
+				auto pm = permanent_alloc_item<lbModule>();
+				pm->file    = f;
+				pm->pkg     = pkg;
+				pm->gen     = gen;
+				pm->checker = c;
+				group->polymorphic_module = pm;
+				pm->polymorphic_module    = pm;
+				map_set(&gen->modules, cast(void *)pm, pm);
+				lb_init_module(pm, do_threading);
+			}
+			group_size = 0;
+		}
+		map_set(&gen->file_modules, f, group);
+		i64 *size = map_get(file_sizes, f);
+		group_size += size ? *size : 0;
+	}
+}
+
 gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) {
 	if (global_error_collector.count != 0) {
 		return false;
@@ -206,9 +287,17 @@ gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) {
 
 	map_init(&gen->modules, gen->info->packages.count*2);
 	map_init(&gen->modules_through_ctx, gen->info->packages.count*2);
+	map_init(&gen->file_modules);
 
 	if (USE_SEPARATE_MODULES) {
 		bool module_per_file = build_context.module_per_file && (build_context.optimization_level <= 0 || build_context.lto_kind != LTO_None);
+		// NOTE(bill): A package with much more code than an even share of all of it is generated in several modules
+		// of contiguous files, as each module is generated by one thread. The share does not depend on the number of
+		// threads, so neither do the modules.
+		PtrMap<AstFile *, i64> file_sizes = {};
+		map_init(&file_sizes, 1024);
+		i64 share = gb_max(lb_estimate_file_code_sizes(gen->info, &file_sizes) / 32, 1);
+
 		for (auto const &entry : gen->info->packages) {
 			AstPackage *pkg = entry.value;
 			auto m = permanent_alloc_item<lbModule>();
@@ -232,6 +321,9 @@ gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) {
 			}
 
 			bool allow_for_per_file = pkg->kind == Package_Runtime || module_per_file;
+			if (!allow_for_per_file) {
+				lb_split_package_into_modules(gen, c, pkg, &file_sizes, share, do_threading);
+			}
 
 			#if 0
 			if (!allow_for_per_file) {
@@ -572,6 +664,10 @@ gb_internal lbModule *lb_module_of_expr(lbGenerator *gen, Ast *expr) {
 		if (found) {
 			return *found;
 		}
+		found = map_get(&gen->file_modules, file);
+		if (found) {
+			return *found;
+		}
 
 		if (file->pkg) {
 			found = map_get(&gen->modules, cast(void *)file->pkg);
@@ -599,6 +695,11 @@ gb_internal lbModule *lb_module_of_entity_internal(lbGenerator *gen, Entity *e,
 			GB_ASSERT(*found != nullptr);
 			return *found;
 		}
+		found = map_get(&gen->file_modules, e->file);
+		if (found) {
+			GB_ASSERT(*found != nullptr);
+			return *found;
+		}
 	}
 	if (e->pkg) {
 		found = map_get(&gen->modules, cast(void *)e->pkg);