diff --git a/.github/actions/install/action.yml b/.github/actions/install/action.yml index 2b8f76dc..0e285631 100644 --- a/.github/actions/install/action.yml +++ b/.github/actions/install/action.yml @@ -19,6 +19,10 @@ runs: run: | ln -s /usr/local/lib/libiconv vendor/libiconv + - name: build mimalloc + shell: bash + run: make install-mimalloc + - name: build netsurf shell: bash run: make install-netsurf diff --git a/.gitmodules b/.gitmodules index 9172f105..3aa7f9fb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,10 +3,10 @@ url = git@github.com:lightpanda-io/jsruntime-lib.git [submodule "vendor/netsurf/libwapcaplet"] path = vendor/netsurf/libwapcaplet - url = https://source.netsurf-browser.org/libwapcaplet.git + url = git@github.com:lightpanda-io/libwapcaplet.git [submodule "vendor/netsurf/libparserutils"] path = vendor/netsurf/libparserutils - url = https://source.netsurf-browser.org/libparserutils.git + url = git@github.com:lightpanda-io/libparserutils.git [submodule "vendor/netsurf/libdom"] path = vendor/netsurf/libdom url = git@github.com:lightpanda-io/libdom.git @@ -15,7 +15,10 @@ url = https://source.netsurf-browser.org/buildsystem.git [submodule "vendor/netsurf/libhubbub"] path = vendor/netsurf/libhubbub - url = https://source.netsurf-browser.org/libhubbub.git + url = git@github.com:lightpanda-io/libhubbub.git [submodule "tests/wpt"] path = tests/wpt url = https://github.com/lightpanda-io/wpt +[submodule "vendor/mimalloc"] + path = vendor/mimalloc + url = git@github.com:microsoft/mimalloc.git diff --git a/Makefile b/Makefile index 8c44a36e..42ee34f7 100644 --- a/Makefile +++ b/Makefile @@ -95,13 +95,14 @@ test: .PHONY: install-submodule .PHONY: install-jsruntime install-jsruntime-dev install-libiconv .PHONY: _install-netsurf install-netsurf clean-netsurf test-netsurf install-netsurf-dev +.PHONY: install-mimalloc install-mimalloc-dev clean-mimalloc .PHONY: install-dev install ## Install and build dependencies for release -install: install-submodule install-jsruntime install-netsurf +install: install-submodule install-jsruntime install-netsurf install-mimalloc ## Install and build dependencies for dev -install-dev: install-submodule install-jsruntime-dev install-netsurf-dev +install-dev: install-submodule install-jsruntime-dev install-netsurf-dev install-mimalloc-dev install-netsurf-dev: _install-netsurf install-netsurf-dev: OPTCFLAGS := -O0 -g -DNDEBUG @@ -185,6 +186,26 @@ install-jsruntime: @cd vendor/jsruntime-lib && \ make install +.PHONY: _build_mimalloc +_build_mimalloc: + @cd vendor/mimalloc && \ + mkdir -p out/include && \ + cp include/mimalloc.h out/include/ && \ + cd out && \ + cmake -DMI_BUILD_SHARED=OFF -DMI_BUILD_OBJECT=OFF -DMI_BUILD_TESTS=OFF -DMI_OVERRIDE=OFF $(OPTS) .. && \ + make + +install-mimalloc-dev: _build_mimalloc +install-mimalloc-dev: OPTS=-DCMAKE_BUILD_TYPE=Debug +install-mimalloc-dev: + @cd vendor/mimalloc/out && \ + mv libmimalloc-debug.a libmimalloc.a + +install-mimalloc: _build_mimalloc + +clean-mimalloc: + @rm -fr vendor/mimalloc/lib/* + ## Init and update git submodule install-submodule: @git submodule init && \ diff --git a/README.md b/README.md index a78620ae..f4054973 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,9 @@ Browsercore is written with [Zig](https://ziglang.org/) `0.12`. You have to install it with the right version in order to build the project. Browsercore also depends on -[js-runtimelib](https://github.com/francisbouvier/jsruntime-lib/) and -[Netsurf libs](https://www.netsurf-browser.org/) libs. +[js-runtimelib](https://github.com/francisbouvier/jsruntime-lib/), +[Netsurf libs](https://www.netsurf-browser.org/) and +[Mimalloc](https://microsoft.github.io/mimalloc) libs. To be able to build the v8 engine for js-runtimelib, you have to install some libs: @@ -36,11 +37,26 @@ make install-submodule ### Build Netsurf -The command `make install-netsurf` will build netsurf libs used by browsercore. +The command `make install-netsurf` will build Netsurf libs used by browsercore. ``` make install-netsurf ``` +For dev env, use `make install-netsurf-dev`. + +### Build Mimalloc + +The command `make install-mimalloc` will build Mimalloc lib used by browsercore. +``` +make install-mimalloc +``` + +For dev env, use `make install-mimalloc-dev`. + +Note, when Mimalloc is built in dev mode, you can dump memory stats with the +env var `MIMALLOC_SHOW_STATS=1`. See +https://microsoft.github.io/mimalloc/environment.html + ### Build jsruntime-lib The command `make install-jsruntime-dev` uses jsruntime-lib's `zig-v8` dependency to build v8 engine lib. diff --git a/build.zig b/build.zig index a414b2d6..c5b8be86 100644 --- a/build.zig +++ b/build.zig @@ -152,6 +152,10 @@ fn common( ) !void { try jsruntime_pkgs.add(step, options); linkNetSurf(step); + + // link mimalloc + step.addObjectFile(.{ .path = "vendor/mimalloc/out/libmimalloc.a" }); + step.addIncludePath(.{ .path = "vendor/mimalloc/out/include" }); } fn linkNetSurf(step: *std.build.LibExeObjStep) void { diff --git a/src/browser/browser.zig b/src/browser/browser.zig index 6cd043c6..0508a391 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -134,6 +134,9 @@ pub const Page = struct { self.session.env.stop(); // TODO unload document: https://html.spec.whatwg.org/#unloading-documents + // clear netsurf memory arena. + parser.deinit(); + _ = self.arena.reset(.free_all); } @@ -211,6 +214,9 @@ pub const Page = struct { fn loadHTMLDoc(self: *Page, reader: anytype, charset: []const u8) !void { const alloc = self.arena.allocator(); + // start netsurf memory arena. + try parser.init(); + log.debug("parse html with charset {s}", .{charset}); const ccharset = try alloc.dupeZ(u8, charset); diff --git a/src/main.zig b/src/main.zig index 1dc34dc6..2d6aa4bc 100644 --- a/src/main.zig +++ b/src/main.zig @@ -56,6 +56,9 @@ pub fn main() !void { var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena.deinit(); + try parser.init(); + defer parser.deinit(); + // document const file = try std.fs.cwd().openFile("test.html", .{}); defer file.close(); diff --git a/src/main_get.zig b/src/main_get.zig index 33a7d11e..de6191c6 100644 --- a/src/main_get.zig +++ b/src/main_get.zig @@ -65,8 +65,10 @@ pub fn main() !void { defer browser.deinit(); var page = try browser.currentSession().createPage(); - defer page.end(); + defer page.deinit(); + try page.navigate(url); + defer page.end(); if (dump) { try page.dump(std.io.getStdOut()); diff --git a/src/main_shell.zig b/src/main_shell.zig index 965e7cf6..43271b3d 100644 --- a/src/main_shell.zig +++ b/src/main_shell.zig @@ -37,6 +37,9 @@ pub fn main() !void { var arena = std.heap.ArenaAllocator.init(gpa.allocator()); defer arena.deinit(); + try parser.init(); + defer parser.deinit(); + // document const file = try std.fs.cwd().openFile("test.html", .{}); defer file.close(); diff --git a/src/mimalloc.zig b/src/mimalloc.zig new file mode 100644 index 00000000..58f98780 --- /dev/null +++ b/src/mimalloc.zig @@ -0,0 +1,58 @@ +// This file makes the glue between mimalloc heap allocation and libdom memory +// management. +// We replace the libdom default usage of allocations with mimalloc heap +// allocation to be able to free all memory used at once, like an arena usage. + +const std = @import("std"); +const c = @cImport({ + @cInclude("mimalloc.h"); +}); + +const Error = error{ + HeapNotNull, + HeapNull, +}; + +var heap: ?*c.mi_heap_t = null; + +pub fn create() Error!void { + if (heap != null) return Error.HeapNotNull; + heap = c.mi_heap_new(); + if (heap == null) return Error.HeapNull; +} + +pub fn destroy() void { + if (heap == null) return; + c.mi_heap_destroy(heap.?); + heap = null; +} + +pub export fn m_alloc(size: usize) callconv(.C) ?*anyopaque { + if (heap == null) return null; + return c.mi_heap_malloc(heap.?, size); +} + +pub export fn re_alloc(ptr: ?*anyopaque, size: usize) callconv(.C) ?*anyopaque { + if (heap == null) return null; + return c.mi_heap_realloc(heap.?, ptr, size); +} + +pub export fn c_alloc(nmemb: usize, size: usize) callconv(.C) ?*anyopaque { + if (heap == null) return null; + return c.mi_heap_calloc(heap.?, nmemb, size); +} + +pub export fn str_dup(s: [*c]const u8) callconv(.C) [*c]u8 { + if (heap == null) return null; + return c.mi_heap_strdup(heap.?, s); +} + +pub export fn strn_dup(s: [*c]const u8, size: usize) callconv(.C) [*c]u8 { + if (heap == null) return null; + return c.mi_heap_strndup(heap.?, s, size); +} + +// NOOP, use destroy to clear all the memory allocated at once. +pub export fn f_ree(_: ?*anyopaque) callconv(.C) void { + return; +} diff --git a/src/netsurf.zig b/src/netsurf.zig index 619c37c7..1bc48e16 100644 --- a/src/netsurf.zig +++ b/src/netsurf.zig @@ -8,9 +8,29 @@ const c = @cImport({ @cInclude("events/event.h"); }); +const mimalloc = @import("mimalloc.zig"); + const Callback = @import("jsruntime").Callback; const EventToInterface = @import("events/event.zig").Event.toInterface; +// init initializes netsurf lib. +// init starts a mimalloc heap arena for the netsurf session. The caller must +// call deinit() to free the arena memory. +pub fn init() !void { + try mimalloc.create(); +} + +// deinit frees the mimalloc heap arena memory. +// It also clean dom namespaces and lwc strings. +pub fn deinit() void { + _ = c.dom_namespace_finalise(); + + // destroy all lwc strings. + c.lwc_deinit_strings(); + + mimalloc.destroy(); +} + // Vtable // ------ diff --git a/src/run_tests.zig b/src/run_tests.zig index bd22965d..14a235b7 100644 --- a/src/run_tests.zig +++ b/src/run_tests.zig @@ -38,6 +38,9 @@ fn testExecFn( js_env: *jsruntime.Env, comptime execFn: jsruntime.ContextExecFn, ) anyerror!void { + try parser.init(); + defer parser.deinit(); + // start JS env try js_env.start(alloc); defer js_env.stop(); @@ -155,6 +158,9 @@ pub fn main() !void { if (run == .all or run == .unit) { std.debug.print("\n", .{}); for (builtin.test_functions) |test_fn| { + try parser.init(); + defer parser.deinit(); + try test_fn.func(); std.debug.print("{s}\tOK\n", .{test_fn.name}); } diff --git a/src/wpt/run.zig b/src/wpt/run.zig index 686363c0..467436a5 100644 --- a/src/wpt/run.zig +++ b/src/wpt/run.zig @@ -17,6 +17,8 @@ const Types = @import("../main_wpt.zig").Types; // It loads first the js libs files. pub fn run(arena: *std.heap.ArenaAllocator, comptime dir: []const u8, f: []const u8, loader: *FileLoader) !jsruntime.JSResult { const alloc = arena.allocator(); + try parser.init(); + defer parser.deinit(); // document const file = try std.fs.cwd().openFile(f, .{}); diff --git a/tests/wpt b/tests/wpt index 46948926..735b2182 160000 --- a/tests/wpt +++ b/tests/wpt @@ -1 +1 @@ -Subproject commit 4694892636c99fac364a203fd3b335dd7199247c +Subproject commit 735b21823e1d3e59f4ff1946612293be196dfc36 diff --git a/vendor/mimalloc b/vendor/mimalloc new file mode 160000 index 00000000..8f7d1e9a --- /dev/null +++ b/vendor/mimalloc @@ -0,0 +1 @@ +Subproject commit 8f7d1e9a41bb0182166aac6a8d4d8b00f60ed032 diff --git a/vendor/netsurf/libdom b/vendor/netsurf/libdom index 9660f919..293790bb 160000 --- a/vendor/netsurf/libdom +++ b/vendor/netsurf/libdom @@ -1 +1 @@ -Subproject commit 9660f919001d4355a46e7c9bbfde11093fc44ba2 +Subproject commit 293790bb9336961a5c7b0e38fd9f254146eee8d0 diff --git a/vendor/netsurf/libhubbub b/vendor/netsurf/libhubbub index 873ed6e2..6f102212 160000 --- a/vendor/netsurf/libhubbub +++ b/vendor/netsurf/libhubbub @@ -1 +1 @@ -Subproject commit 873ed6e236f7669afd3ef44259c34addc6dc95b6 +Subproject commit 6f102212c85f48a3c61c916ea46a74a37a2bfc9b diff --git a/vendor/netsurf/libparserutils b/vendor/netsurf/libparserutils index 96cdd0ff..094dc22e 160000 --- a/vendor/netsurf/libparserutils +++ b/vendor/netsurf/libparserutils @@ -1 +1 @@ -Subproject commit 96cdd0ff114299f520e76538ab8fde39358b87f9 +Subproject commit 094dc22e2b3c21e8d12f2275fd7bf09bc4da3f3e diff --git a/vendor/netsurf/libwapcaplet b/vendor/netsurf/libwapcaplet index b5e42b12..74f1e011 160000 --- a/vendor/netsurf/libwapcaplet +++ b/vendor/netsurf/libwapcaplet @@ -1 +1 @@ -Subproject commit b5e42b12211a92339b0b62cb90f1a86a397e146e +Subproject commit 74f1e0117310b5392da484a71346cf09f78e8216