From c55afc1df9b38d227dfd2e5c0c8e298f24227131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 9 Sep 2026 17:49:02 +0200 Subject: [PATCH] cli: add --locale and --timezone, derive language signals from one value navigator.language was hard-coded to en-US and Accept-Language was a constant, while Intl, toLocaleString and Date followed the host process environment. On a de_DE host a page saw navigator.language === "en-US" next to German number formatting, a mismatch fingerprinting scripts look for, and the same page rendered differently across machines. Follow Chrome's --lang rule: one configured tag drives navigator.language(s), the Accept-Language header and ICU's default locale. --locale defaults to en-US, so Intl is now en-US on every host instead of whatever LANG says. --timezone sets the IANA zone Date and Intl use; absent, the host zone stays. Both are applied by writing LC_ALL and TZ before V8 initializes ICU, which reads them lazily. Platform.init is the first call in App.init, before any thread exists, so setenv is safe there. CDP Emulation.setUserAgentOverride.acceptLanguage, which Playwright sends for its locale option, now overrides the header and navigator.languages for the browser context's lifetime, mirroring the user agent override, and applies even when the Mozilla user agent is refused. Emulation.setLocaleOverride and setTimezoneOverride stay no-ops: changing ICU's defaults at runtime needs new zig-v8-fork bindings. --- src/App.zig | 6 +- src/Config.zig | 203 ++++++++++++++++++++++++- src/browser/js/Platform.zig | 64 +++++++- src/browser/webapi/Navigator.zig | 9 +- src/browser/webapi/WorkerNavigator.zig | 8 +- src/help.zon | 7 + src/main_snapshot_creator.zig | 2 +- src/network/HttpClient.zig | 101 +++++++++++- src/server/cdp/CDP.zig | 4 + src/server/cdp/domains/emulation.zig | 87 +++++++++-- 10 files changed, 468 insertions(+), 23 deletions(-) diff --git a/src/App.zig b/src/App.zig index 484b67984..fe5a0effa 100644 --- a/src/App.zig +++ b/src/App.zig @@ -44,7 +44,11 @@ arena_pool: ArenaPool, app_dir_path: ?[]const u8, pub fn init(allocator: Allocator, config: *const Config) !*App { - const platform = try Platform.init(config.v8Flags()); + const platform = try Platform.init(.{ + .v8_flags = config.v8Flags(), + .locale = config.locale(), + .timezone = config.timezone(), + }); errdefer platform.deinit(); const snapshot = try Snapshot.load(); diff --git a/src/Config.zig b/src/Config.zig index 031becf6b..9db44ba42 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -276,6 +276,8 @@ const CommonOptions = .{ .{ .name = "web_bot_auth_keyid", .type = ?[]const u8 }, .{ .name = "web_bot_auth_domain", .type = ?[]const u8 }, .{ .name = "user_agent", .type = ?[]const u8, .validator = userAgentValidator }, + .{ .name = "locale", .type = []const u8, .default = HttpHeaders.default_locale, .validator = localeValidator }, + .{ .name = "timezone", .type = ?[]const u8, .validator = timezoneValidator }, .{ .name = "block_private_networks", .type = bool }, .{ .name = "block_cidrs", .type = ?[]const u8, .validator = accumulateValidator }, .{ .name = "block_urls", .type = ?[]const u8, .validator = accumulateValidator }, @@ -727,6 +729,20 @@ pub fn userAgent(self: *const Config) ?[]const u8 { }; } +pub fn locale(self: *const Config) []const u8 { + return switch (self.mode) { + inline .serve, .fetch, .mcp, .agent => |opts| opts.locale, + else => HttpHeaders.default_locale, + }; +} + +pub fn timezone(self: *const Config) ?[]const u8 { + return switch (self.mode) { + inline .serve, .fetch, .mcp, .agent => |opts| opts.timezone, + else => null, + }; +} + pub fn httpCacheDir(self: *const Config) ?[]const u8 { return switch (self.mode) { inline .serve, .fetch, .mcp, .agent => |opts| opts.http_cache_dir, @@ -971,7 +987,8 @@ pub const HttpHeaders = struct { // stream when a client sends Accept-Encoding without Accept-Language, // treating it as a bot signal. Ship a neutral default so we look like a // normal client. - pub const accept_language: [:0]const u8 = "en-US,en;q=0.9"; + pub const default_locale: []const u8 = "en-US"; + pub const accept_language_default: [:0]const u8 = "en-US,en;q=0.9"; // Document-navigation Accept value Chrome sends. pub const navigation_accept: [:0]const u8 = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; @@ -980,6 +997,12 @@ pub const HttpHeaders = struct { proxy_bearer_header: ?[:0]const u8, + // Derived from --locale: the header and the navigator.languages list are + // the same information in two shapes, so they are built together. + accept_language: [:0]const u8, + languages: [2][]const u8, + languages_len: u8, + pub fn init(allocator: Allocator, config: *const Config) !HttpHeaders { const user_agent: [:0]const u8 = if (config.userAgent()) |ua| try allocator.dupeZ(u8, ua) @@ -993,10 +1016,18 @@ pub const HttpHeaders = struct { try std.fmt.allocPrintSentinel(allocator, "Proxy-Authorization: Bearer {s}", .{token}, 0) else null; + errdefer if (proxy_bearer_header) |hdr| allocator.free(hdr); + + const tag = config.locale(); + const primary = primarySubtag(tag); + const accept_language = try acceptLanguageFor(allocator, tag); return .{ .user_agent = user_agent, .proxy_bearer_header = proxy_bearer_header, + .accept_language = accept_language, + .languages = .{ tag, primary }, + .languages_len = if (primary.len == tag.len) 1 else 2, }; } @@ -1007,6 +1038,36 @@ pub const HttpHeaders = struct { if (self.user_agent.ptr != user_agent_base.ptr) { allocator.free(self.user_agent); } + if (self.accept_language.ptr != accept_language_default.ptr) { + allocator.free(self.accept_language); + } + } + + pub fn primarySubtag(tag: []const u8) []const u8 { + const end = std.mem.indexOfScalar(u8, tag, '-') orelse tag.len; + return tag[0..end]; + } + + /// Chrome's shape: the tag first, then its language alone, then English + /// as a last resort, with descending q values. The default returns the + /// comptime constant so callers can tell it apart by pointer. + pub fn acceptLanguageFor(allocator: Allocator, tag: []const u8) ![:0]const u8 { + if (std.mem.eql(u8, tag, default_locale)) { + return accept_language_default; + } + const primary = primarySubtag(tag); + const has_region = primary.len != tag.len; + const is_english = std.ascii.eqlIgnoreCase(primary, "en"); + if (has_region and !is_english) { + return std.fmt.allocPrintSentinel(allocator, "{s},{s};q=0.9,en;q=0.8", .{ tag, primary }, 0); + } + if (has_region) { + return std.fmt.allocPrintSentinel(allocator, "{s},{s};q=0.9", .{ tag, primary }, 0); + } + if (!is_english) { + return std.fmt.allocPrintSentinel(allocator, "{s},en;q=0.9", .{tag}, 0); + } + return allocator.dupeZ(u8, tag); } }; @@ -1245,6 +1306,84 @@ test "Config: validateUserAgent" { try std.testing.expectError(error.NonPrintable, validateUserAgent("bad\x01ua")); } +test "Config: validateLocale" { + try validateLocale("en"); + try validateLocale("en-US"); + try validateLocale("zh-Hant-TW"); + try validateLocale("es-419"); + try std.testing.expectError(error.InvalidLanguage, validateLocale("")); + try std.testing.expectError(error.InvalidLanguage, validateLocale("e")); + try std.testing.expectError(error.InvalidLanguage, validateLocale("en_US")); + try std.testing.expectError(error.InvalidSubtag, validateLocale("en-")); + try std.testing.expectError(error.InvalidSubtag, validateLocale("en-U")); + try std.testing.expectError(error.InvalidSubtag, validateLocale("en-US-x-toolongsub")); + try std.testing.expectError(error.InvalidSubtag, validateLocale("en-U$")); + try std.testing.expectError(error.TooLong, validateLocale("en-" ++ "a" ** 40)); +} + +test "Config: validateTimezone" { + try validateTimezone("UTC"); + try validateTimezone("Europe/Paris"); + try validateTimezone("America/Argentina/Buenos_Aires"); + try std.testing.expectError(error.Empty, validateTimezone("")); + try std.testing.expectError(error.InvalidCharacter, validateTimezone("Europe/ Paris")); + try std.testing.expectError(error.InvalidCharacter, validateTimezone("UTC\n")); + try std.testing.expectError(error.TooLong, validateTimezone("a" ** 65)); +} + +test "Config: HttpHeaders.acceptLanguageFor" { + const allocator = std.testing.allocator; + const cases = [_]struct { tag: []const u8, expected: []const u8 }{ + .{ .tag = "en-US", .expected = "en-US,en;q=0.9" }, + .{ .tag = "en-GB", .expected = "en-GB,en;q=0.9" }, + .{ .tag = "de-DE", .expected = "de-DE,de;q=0.9,en;q=0.8" }, + .{ .tag = "de", .expected = "de,en;q=0.9" }, + .{ .tag = "en", .expected = "en" }, + .{ .tag = "zh-Hant-TW", .expected = "zh-Hant-TW,zh;q=0.9,en;q=0.8" }, + }; + for (cases) |case| { + const value = try HttpHeaders.acceptLanguageFor(allocator, case.tag); + defer if (value.ptr != HttpHeaders.accept_language_default.ptr) allocator.free(value); + try std.testing.expectEqualStrings(case.expected, value); + } + const default = try HttpHeaders.acceptLanguageFor(allocator, "en-US"); + try std.testing.expectEqual(HttpHeaders.accept_language_default.ptr, default.ptr); +} + +test "Config: locale drives http_headers" { + const allocator = std.testing.allocator; + { + var config = try Config.init(allocator, "test", .{ .serve = .{ .host = "127.0.0.1" } }); + defer config.deinit(allocator); + try std.testing.expectEqualStrings("en-US,en;q=0.9", config.http_headers.accept_language); + try std.testing.expectEqual(2, config.http_headers.languages_len); + try std.testing.expectEqualStrings("en-US", config.http_headers.languages[0]); + try std.testing.expectEqualStrings("en", config.http_headers.languages[1]); + } + { + var config = try Config.init(allocator, "test", .{ .serve = .{ .host = "127.0.0.1", .locale = "fr" } }); + defer config.deinit(allocator); + try std.testing.expectEqualStrings("fr,en;q=0.9", config.http_headers.accept_language); + try std.testing.expectEqual(1, config.http_headers.languages_len); + try std.testing.expectEqualStrings("fr", config.http_headers.languages[0]); + } +} + +test "Config: parseArgs refuses an invalid --locale and --timezone" { + { + log.expectLog(&.{.app}); + const argv = [_][*:0]const u8{ "lightpanda", "fetch", "--locale", "en_US" }; + const proc_args: std.process.Args = .{ .vector = &argv }; + try std.testing.expectError(error.InvalidArgument, parseArgs(std.testing.allocator, proc_args)); + } + { + log.expectLog(&.{.app}); + const argv = [_][*:0]const u8{ "lightpanda", "fetch", "--timezone", "Europe/ Paris" }; + const proc_args: std.process.Args = .{ .vector = &argv }; + try std.testing.expectError(error.InvalidArgument, parseArgs(std.testing.allocator, proc_args)); + } +} + test "Config: parseArgs refuses an invalid --http-header" { const invalid = [_][*:0]const u8{ // no colon to split on @@ -1375,6 +1514,68 @@ pub fn validateUserAgent(ua: []const u8) !void { } } +fn localeValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *[]const u8) !void { + const str = args.next() orelse return error.MissingArgument; + validateLocale(str) catch |err| { + log.fatal(.app, "invalid option value", .{ .arg = "--locale", .value = str, .err = err, .hint = "must be a BCP 47 tag such as en-US, de or zh-Hant-TW" }); + return error.InvalidArgument; + }; + field.* = try allocator.dupe(u8, str); +} + +fn timezoneValidator(allocator: Allocator, args: *std.process.Args.Iterator, field: *?[]const u8) !void { + const str = args.next() orelse return error.MissingArgument; + validateTimezone(str) catch |err| { + log.fatal(.app, "invalid option value", .{ .arg = "--timezone", .value = str, .err = err, .hint = "must be an IANA time zone such as Europe/Paris or UTC" }); + return error.InvalidArgument; + }; + field.* = try allocator.dupe(u8, str); +} + +/// A BCP 47 tag restricted to what ICU and the Accept-Language derivation +/// need: a 2-3 letter language followed by 2-8 character alphanumeric subtags. +pub fn validateLocale(tag: []const u8) !void { + if (tag.len > 35) { + return error.TooLong; + } + var it = std.mem.splitScalar(u8, tag, '-'); + const language = it.next().?; + if (language.len < 2 or language.len > 3) { + return error.InvalidLanguage; + } + for (language) |c| { + if (!std.ascii.isAlphabetic(c)) { + return error.InvalidLanguage; + } + } + while (it.next()) |subtag| { + if (subtag.len < 2 or subtag.len > 8) { + return error.InvalidSubtag; + } + for (subtag) |c| { + if (!std.ascii.isAlphanumeric(c)) { + return error.InvalidSubtag; + } + } + } +} + +/// Only the shape is checked; ICU resolves the id itself and falls back to +/// GMT for names it does not know. +pub fn validateTimezone(id: []const u8) !void { + if (id.len == 0) { + return error.Empty; + } + if (id.len > 64) { + return error.TooLong; + } + for (id) |c| { + if (c <= ' ' or c == 0x7f) { + return error.InvalidCharacter; + } + } +} + /// Tag names of a Zig enum, so a command's allowed values can't drift from the /// enum it sets. pub const tagNames = cli.tagNames; diff --git a/src/browser/js/Platform.zig b/src/browser/js/Platform.zig index ab9a60c08..942b84150 100644 --- a/src/browser/js/Platform.zig +++ b/src/browser/js/Platform.zig @@ -16,17 +16,39 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +const std = @import("std"); const js = @import("js.zig"); const v8 = js.v8; const Platform = @This(); handle: *v8.Platform, -pub fn init(v8_flags: ?[]const u8) !Platform { - if (v8_flags) |flags| { +pub const Options = struct { + v8_flags: ?[]const u8 = null, + // BCP 47 tag; becomes ICU's default locale (Intl, toLocaleString). + locale: ?[]const u8 = null, + // IANA id; becomes ICU's default time zone. Null keeps the host zone. + timezone: ?[]const u8 = null, +}; + +/// ICU reads LC_ALL and TZ lazily on first use, so the environment must be +/// set here, before InitializeICU and before the platform starts its thread +/// pool (setenv is not safe once other threads may call getenv). +pub fn init(opts: Options) !Platform { + if (opts.v8_flags) |flags| { v8.v8__V8__SetFlagsFromString(flags.ptr, flags.len); } + if (opts.locale) |tag| { + var buf: [32]u8 = undefined; + _ = setenv("LC_ALL", posixLocaleId(&buf, tag), 1); + } + if (opts.timezone) |id| { + var buf: [128]u8 = undefined; + const value = std.fmt.bufPrintZ(&buf, "{s}", .{id}) catch return error.TimezoneTooLong; + _ = setenv("TZ", value, 1); + } + if (v8.v8__V8__InitializeICU() == false) { return error.FailedToInitializeICU; } @@ -43,3 +65,41 @@ pub fn deinit(self: Platform) void { v8.v8__V8__DisposePlatform(); v8.v8__Platform__DELETE(self.handle); } + +/// `language[_REGION].UTF-8`, the POSIX id ICU parses from LC_ALL. The region +/// is the first 2-letter or 3-digit subtag, so a script subtag is skipped. +/// The tag is assumed valid per Config.validateLocale (language <= 3 bytes). +pub fn posixLocaleId(buf: *[32]u8, tag: []const u8) [:0]const u8 { + var it = std.mem.splitScalar(u8, tag, '-'); + var language_buf: [3]u8 = undefined; + const language = std.ascii.lowerString(&language_buf, it.next().?); + + while (it.next()) |subtag| { + const is_alpha2 = subtag.len == 2 and std.ascii.isAlphabetic(subtag[0]) and std.ascii.isAlphabetic(subtag[1]); + const is_digit3 = subtag.len == 3 and std.ascii.isDigit(subtag[0]) and std.ascii.isDigit(subtag[1]) and std.ascii.isDigit(subtag[2]); + if (is_alpha2 or is_digit3) { + var region_buf: [3]u8 = undefined; + const region = std.ascii.upperString(®ion_buf, subtag); + return std.fmt.bufPrintZ(buf, "{s}_{s}.UTF-8", .{ language, region }) catch unreachable; + } + } + return std.fmt.bufPrintZ(buf, "{s}.UTF-8", .{language}) catch unreachable; +} + +extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, override: c_int) c_int; + +test "Platform: posixLocaleId" { + const cases = [_]struct { tag: []const u8, expected: []const u8 }{ + .{ .tag = "en-US", .expected = "en_US.UTF-8" }, + .{ .tag = "de-DE", .expected = "de_DE.UTF-8" }, + .{ .tag = "en", .expected = "en.UTF-8" }, + .{ .tag = "zh-Hant-TW", .expected = "zh_TW.UTF-8" }, + .{ .tag = "es-419", .expected = "es_419.UTF-8" }, + .{ .tag = "PT-br", .expected = "pt_BR.UTF-8" }, + .{ .tag = "de-DE-1996", .expected = "de_DE.UTF-8" }, + }; + for (cases) |case| { + var buf: [32]u8 = undefined; + try std.testing.expectEqualStrings(case.expected, posixLocaleId(&buf, case.tag)); + } +} diff --git a/src/browser/webapi/Navigator.zig b/src/browser/webapi/Navigator.zig index 4beeae7dc..65fd19d86 100644 --- a/src/browser/webapi/Navigator.zig +++ b/src/browser/webapi/Navigator.zig @@ -53,8 +53,8 @@ pub fn getUserAgent(_: *const Navigator, exec: *const Execution) []const u8 { return exec.session.browser.http_client.getUserAgent(); } -pub fn getLanguages(_: *const Navigator) [2][]const u8 { - return .{ "en-US", "en" }; +pub fn getLanguages(_: *const Navigator, exec: *const Execution) []const []const u8 { + return exec.session.browser.http_client.getLanguages(); } pub fn getDoNotTrack(_: *const Navigator) ?[]const u8 { @@ -73,8 +73,9 @@ pub fn getAppVersion(_: *const Navigator) []const u8 { return "1.0"; } -pub fn getLanguage(_: *const Navigator) []const u8 { - return "en-US"; +pub fn getLanguage(self: *const Navigator, exec: *const Execution) []const u8 { + const languages = self.getLanguages(exec); + return if (languages.len == 0) "" else languages[0]; } pub fn getOnLine(_: *const Navigator) bool { diff --git a/src/browser/webapi/WorkerNavigator.zig b/src/browser/webapi/WorkerNavigator.zig index cced8ae70..7a03cedac 100644 --- a/src/browser/webapi/WorkerNavigator.zig +++ b/src/browser/webapi/WorkerNavigator.zig @@ -46,8 +46,8 @@ pub fn getUserAgent(_: *const WorkerNavigator, exec: *const Execution) []const u return Navigator.getUserAgent(&Navigator.init, exec); } -pub fn getLanguages(_: *const WorkerNavigator) [2][]const u8 { - return Navigator.getLanguages(&Navigator.init); +pub fn getLanguages(_: *const WorkerNavigator, exec: *const Execution) []const []const u8 { + return Navigator.getLanguages(&Navigator.init, exec); } pub fn getAppName(_: *const WorkerNavigator) []const u8 { @@ -62,8 +62,8 @@ pub fn getAppVersion(_: *const WorkerNavigator) []const u8 { return Navigator.getAppVersion(&Navigator.init); } -pub fn getLanguage(_: *const WorkerNavigator) []const u8 { - return Navigator.getLanguage(&Navigator.init); +pub fn getLanguage(_: *const WorkerNavigator, exec: *const Execution) []const u8 { + return Navigator.getLanguage(&Navigator.init, exec); } pub fn getOnLine(_: *const WorkerNavigator) bool { diff --git a/src/help.zon b/src/help.zon index dbb92e69f..809efa3b0 100644 --- a/src/help.zon +++ b/src/help.zon @@ -459,6 +459,10 @@ \\ The log level. \\ Defaults to {1s}. \\ Allowed values: "debug", "info", "warn", "error", "fatal". + \\ --locale + \\ BCP 47 language tag driving navigator.language, the + \\ Accept-Language header and the default locale of Intl and + \\ toLocaleString. Defaults to en-US. \\ --obey-robots \\ Fetches and obeys robots.txt of the target page. \\ Defaults to false. @@ -472,6 +476,9 @@ \\ --storage-sqlite-path \\ Path to the SQLite database file for persistent storage. \\ Use ":memory:" for in-memory storage. + \\ --timezone + \\ Time zone used by Date and Intl, e.g. Europe/Paris or UTC. + \\ Defaults to the host time zone. \\ --user-agent \\ Override the User-Agent header entirely. Must not impersonate other \\ browsers; any value containing "Mozilla" is forbidden. The browser diff --git a/src/main_snapshot_creator.zig b/src/main_snapshot_creator.zig index a5b3736d8..03378b6ad 100644 --- a/src/main_snapshot_creator.zig +++ b/src/main_snapshot_creator.zig @@ -39,7 +39,7 @@ pub fn main(init: std.process.Init) !void { } } - var platform = try lp.js.Platform.init(v8_flags); + var platform = try lp.js.Platform.init(.{ .v8_flags = v8_flags }); defer platform.deinit(); const snapshot = try lp.js.Snapshot.create(); diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index 2ff673156..12ff36afe 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -153,6 +153,10 @@ test_fail_submit: if (lp.IS_TEST) ?anyerror else void = if (lp.IS_TEST) null els // Allocated from self.allocator when set, null otherwise. user_agent_override: ?[:0]const u8 = null, +// Accept-Language override set via CDP Emulation.setUserAgentOverride. +// Drives both the request header and navigator.languages. +accept_language_override: ?AcceptLanguageOverride = null, + // The driver (CDP / BiDi) attached to us. If there's a driver, then there's // an inbox for us to process (and there's someone to wake us up from a poll) driver: ?Driver = null, @@ -263,6 +267,7 @@ pub fn deinit(self: *Client) void { self.handles.deinit(); self.clearUserAgentOverride(); + self.clearAcceptLanguageOverride(); if (self.http_proxy_owned) |owned| { self.allocator.free(owned); } @@ -303,6 +308,52 @@ pub fn clearUserAgentOverride(self: *Client) void { } } +const AcceptLanguageOverride = struct { + // The header as given; `languages` are sub-slices of it. + header: [:0]const u8, + languages: []const []const u8, + + const max_languages = 8; +}; + +/// Set an Accept-Language override, allocated from self.allocator. The +/// navigator.languages list is the header's tags with weights dropped. +pub fn setAcceptLanguageOverride(self: *Client, value: []const u8) !void { + self.clearAcceptLanguageOverride(); + + const header = try self.allocator.dupeZ(u8, value); + errdefer self.allocator.free(header); + + var languages: std.ArrayList([]const u8) = .empty; + errdefer languages.deinit(self.allocator); + + var it = std.mem.splitScalar(u8, header, ','); + while (it.next()) |item| { + if (languages.items.len == AcceptLanguageOverride.max_languages) { + break; + } + const end = std.mem.indexOfScalar(u8, item, ';') orelse item.len; + const tag = std.mem.trim(u8, item[0..end], " \t"); + if (tag.len == 0) { + continue; + } + try languages.append(self.allocator, tag); + } + + self.accept_language_override = .{ + .header = header, + .languages = try languages.toOwnedSlice(self.allocator), + }; +} + +pub fn clearAcceptLanguageOverride(self: *Client) void { + if (self.accept_language_override) |override| { + self.allocator.free(override.languages); + self.allocator.free(override.header); + self.accept_language_override = null; + } +} + // Enable TLS verification on all connections. pub fn setTlsVerify(self: *Client, verify: bool) !void { // Remove inflight connections check on enable TLS b/c chromiumoxide calls @@ -428,6 +479,23 @@ pub fn getUserAgent(self: *const Client) [:0]const u8 { return self.user_agent_override orelse self.network.config.http_headers.user_agent; } +pub fn getAcceptLanguage(self: *const Client) [:0]const u8 { + if (self.accept_language_override) |override| { + return override.header; + } + return self.network.config.http_headers.accept_language; +} + +/// What navigator.languages reports: the override's tags when set, else the +/// list derived from --locale. +pub fn getLanguages(self: *const Client) []const []const u8 { + if (self.accept_language_override) |override| { + return override.languages; + } + const headers = &self.network.config.http_headers; + return headers.languages[0..headers.languages_len]; +} + // Headers _all_ requests include. pub fn baselineHeaders(self: *const Client) [4]Transfer.RequestHeader { return .{ @@ -436,7 +504,7 @@ pub fn baselineHeaders(self: *const Client) [4]Transfer.RequestHeader { .{ .name = "Sec-Ch-Ua-Full-Version-List", .value = lp.Config.HttpHeaders.sec_ch_ua_full_version_list, .source = .fixed }, // Omitting Accept-Language triggers bot-protection on some CDNs // (Akamai) when Accept-Encoding is present. - .{ .name = "Accept-Language", .value = lp.Config.HttpHeaders.accept_language }, + .{ .name = "Accept-Language", .value = self.getAcceptLanguage() }, }; } @@ -4203,6 +4271,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void { .single_flight = .init(testing.allocator), }; client.url_blocklist = null; + client.accept_language_override = null; client.test_fail_submit = null; // isUrlBlocked reaches through here for the adblocker; tests that want // one assign it to `client.network` after this returns. @@ -4231,6 +4300,36 @@ test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" { try testing.expectEqual(null, client.url_blocklist); } +test "HttpClient: setAcceptLanguageOverride owns, parses, replaces, and clears" { + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: Client = undefined; + initTestClient(&client, &pool); + defer client.clearAcceptLanguageOverride(); + + var first = "de-DE,de;q=0.9, en;q=0.8".*; + try client.setAcceptLanguageOverride(&first); + @memset(&first, 'x'); + + try std.testing.expectEqualStrings("de-DE,de;q=0.9, en;q=0.8", client.getAcceptLanguage()); + try testing.expectEqual(3, client.getLanguages().len); + try std.testing.expectEqualStrings("de-DE", client.getLanguages()[0]); + try std.testing.expectEqualStrings("de", client.getLanguages()[1]); + try std.testing.expectEqualStrings("en", client.getLanguages()[2]); + + try client.setAcceptLanguageOverride("fr-FR"); + try testing.expectEqual(1, client.getLanguages().len); + try std.testing.expectEqualStrings("fr-FR", client.getLanguages()[0]); + + try client.setAcceptLanguageOverride(""); + try std.testing.expectEqualStrings("", client.getAcceptLanguage()); + try testing.expectEqual(0, client.getLanguages().len); + + client.clearAcceptLanguageOverride(); + try testing.expectEqual(null, client.accept_language_override); +} + const TestRequest = struct { url: [:0]const u8, document: [:0]const u8 = "", diff --git a/src/server/cdp/CDP.zig b/src/server/cdp/CDP.zig index 4785f787a..e2be5d3f1 100644 --- a/src/server/cdp/CDP.zig +++ b/src/server/cdp/CDP.zig @@ -478,6 +478,7 @@ pub const BrowserContext = struct { http_proxy_changed: bool = false, user_agent_changed: bool = false, + accept_language_changed: bool = false, // Extra headers to add to all requests. extra_headers: std.ArrayList(http.Header) = .empty, @@ -627,6 +628,9 @@ pub const BrowserContext = struct { if (self.user_agent_changed) { browser.http_client.clearUserAgentOverride(); } + if (self.accept_language_changed) { + browser.http_client.clearAcceptLanguageOverride(); + } self.intercept_state.deinit(); } diff --git a/src/server/cdp/domains/emulation.zig b/src/server/cdp/domains/emulation.zig index 37f3aae93..a841023a6 100644 --- a/src/server/cdp/domains/emulation.zig +++ b/src/server/cdp/domains/emulation.zig @@ -21,6 +21,7 @@ const lp = @import("lightpanda"); const CDP = @import("../CDP.zig"); const Config = @import("../../../Config.zig"); +const Mime = @import("../../../browser/Mime.zig"); const js = @import("../../../browser/js/js.zig"); const log = lp.log; @@ -157,9 +158,6 @@ pub fn setUserAgentOverride(cmd: *CDP.Command) !void { platform: ?[]const u8 = null, })) orelse return error.InvalidParams; - if (params.acceptLanguage) |v| { - log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "acceptLanguage", .value = v }); - } if (params.platform) |v| { log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "platform", .value = v }); } @@ -167,14 +165,27 @@ pub fn setUserAgentOverride(cmd: *CDP.Command) !void { const ua = params.userAgent; Config.validateUserAgent(ua) catch |err| switch (err) { error.NonPrintable => return cmd.sendError(-32602, "User agent contains non-printable characters", .{}), - error.Reserved => { - log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "userAgent", .value = ua, .info = "User agent must not contain Mozilla" }); - return cmd.sendResult(null, .{}); - }, + error.Reserved => {}, }; const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const http_client = &cmd.cdp.browser.http_client; + + // Applied even when the user agent is refused below: Playwright always + // sends a Mozilla user agent alongside the locale it was asked for. + if (params.acceptLanguage) |accept_language| { + if (!Mime.isHttpHeaderValue(accept_language)) { + return cmd.sendError(-32602, "Accept-Language contains CR, LF or NUL", .{}); + } + try http_client.setAcceptLanguageOverride(accept_language); + bc.accept_language_changed = true; + } + + if (std.ascii.indexOfIgnoreCase(ua, "mozilla") != null) { + log.warn(.not_implemented, "Emulation.setUserAgentOverride", .{ .param = "userAgent", .value = ua, .info = "User agent must not contain Mozilla" }); + return cmd.sendResult(null, .{}); + } + try http_client.setUserAgentOverride(ua); bc.user_agent_changed = true; @@ -211,8 +222,10 @@ fn clearGeolocationOverride(cmd: *CDP.Command) !void { return cmd.sendResult(null, .{}); } -// Accepted so drivers can finish context setup; Intl/Date keep the host's -// locale and timezone. +// Accepted so drivers can finish context setup; Intl/Date keep the process +// locale and timezone (--locale, --timezone). Changing them at runtime needs +// zig-v8-fork bindings for Isolate::DateTimeConfigurationChangeNotification +// and the ICU default locale. fn setLocaleOverride(cmd: *CDP.Command) !void { const Params = struct { locale: ?[]const u8 = null }; const params = (try cmd.params(Params)) orelse Params{}; @@ -381,6 +394,62 @@ test "cdp.Emulation: setUserAgentOverride with optional params" { try ctx.expectSentResult(null, .{ .id = 5 }); } +test "cdp.Emulation: setUserAgentOverride acceptLanguage drives navigator.languages" { + testing.silenceLog(&.{.not_implemented}); + + var ctx = try testing.context(); + defer ctx.deinit(); + var bc = try ctx.loadBrowserContext(.{ .id = "BID-AL", .url = "hi.html", .target_id = "FID-00000000AL".* }); + const frame = bc.mainFrame() orelse unreachable; + + // The default locale reaches both navigator and ICU. + try expectJs(frame, "navigator.language === 'en-US' && navigator.languages.join() === 'en-US,en'"); + try expectJs(frame, "Intl.DateTimeFormat().resolvedOptions().locale === navigator.language"); + + try ctx.processMessage(.{ + .id = 1, + .method = "Emulation.setUserAgentOverride", + .params = .{ .userAgent = "CustomBot/2.0", .acceptLanguage = "de-DE,de;q=0.9, en;q=0.8" }, + }); + try ctx.expectSentResult(null, .{ .id = 1 }); + try expectJs(frame, "navigator.language === 'de-DE' && navigator.languages.join() === 'de-DE,de,en'"); + try std.testing.expectEqualStrings("de-DE,de;q=0.9, en;q=0.8", ctx.cdp().browser.http_client.getAcceptLanguage()); + try testing.expect(bc.accept_language_changed); + + // A Mozilla user agent is refused, the language still applies. + try ctx.processMessage(.{ + .id = 2, + .method = "Emulation.setUserAgentOverride", + .params = .{ .userAgent = "Mozilla/5.0", .acceptLanguage = "fr-FR" }, + }); + try ctx.expectSentResult(null, .{ .id = 2 }); + try expectJs(frame, "navigator.language === 'fr-FR' && navigator.languages.length === 1"); + try std.testing.expectEqualStrings("CustomBot/2.0", ctx.cdp().browser.http_client.getUserAgent()); + + try ctx.processMessage(.{ + .id = 3, + .method = "Emulation.setUserAgentOverride", + .params = .{ .userAgent = "CustomBot/2.0", .acceptLanguage = "" }, + }); + try ctx.expectSentResult(null, .{ .id = 3 }); + try expectJs(frame, "navigator.language === '' && navigator.languages.length === 0"); + + try ctx.processMessage(.{ + .id = 4, + .method = "Emulation.setUserAgentOverride", + .params = .{ .userAgent = "CustomBot/2.0", .acceptLanguage = "en\r\nX-Injected: 1" }, + }); + try ctx.expectSentError(-32602, "Accept-Language contains CR, LF or NUL", .{ .id = 4 }); +} + +fn expectJs(frame: *lp.Frame, expression: [:0]const u8) !void { + var ls: js.Local.Scope = undefined; + frame.js.localScope(&ls); + defer ls.deinit(); + const value = try ls.local.exec(expression, null); + try testing.expect(value.toBool()); +} + test "cdp.Emulation: setUserAgentOverride can be called multiple times" { var ctx = try testing.context(); defer ctx.deinit();