mirror/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-07-31 09:46:05 -04:00
Code Issues Packages Projects Releases Wiki Activity

agent: highlight markdown code blocks as JS

Browse Source 
Extracts the JavaScript highlighting logic from `Terminal.zig` into a
reusable `js_highlight.zig` module. Uses this module to syntax-highlight
markdown fenced code blocks in `md_term.zig` using ANSI escape codes.
This commit is contained in:
Adrià Arrufat
2026-07-15 09:10:11 +02:00
parent 3083414798
commit 8b1686f6fb
3 changed files with 348 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
src/agent/Terminal.zig
View File

@@ -25,6 +25,7 @@ const Schema = lp.Schema;
const SlashCommand = @import("SlashCommand.zig");
const Spinner = @import("Spinner.zig");
const md_term = @import("md_term.zig");
const js_highlight = @import("js_highlight.zig");
const c = @cImport({
@cInclude("isocline.h");
});
@@ -813,19 +814,9 @@ fn highlightBareToken(henv: ?*c.ic_highlight_env_t, text: []const u8, start: usi
} else |_| {}
}
/// Returns the index just past the matching closing quote, or `text.len` if
/// unterminated. Does not handle backslash escapes (matches Schema.tokenize).
/// Index just past the matching closing quote, or `text.len` if unterminated.
fn scanQuoted(text: []const u8, start: usize) usize {
if (start >= text.len) return start;
const ch = text[start];
const is_triple = start + 2 < text.len and text[start + 1] == ch and text[start + 2] == ch;
if (is_triple) {
const triple_delim = text[start .. start + 3];
const close = std.mem.indexOfPos(u8, text, start + 3, triple_delim) orelse return text.len;
return close + 3;
}
const close = std.mem.indexOfScalarPos(u8, text, start + 1, ch) orelse return text.len;
return close + 1;
return js_highlight.scanString(text, start).end;
}
/// Highlight `$LP_*` tokens appearing from `start` onward.
@@ -853,95 +844,29 @@ fn highlightDollarVarsIn(henv: ?*c.ic_highlight_env_t, text: []const u8, start:
}
}
const js_keywords = [_][]const u8{
"function", "async", "await", "yield", "return", "if", "else",
"for", "while", "do", "switch", "case", "break", "continue",
"var", "let", "const", "new", "delete", "typeof", "instanceof",
"in", "of", "void", "this", "super", "class", "extends",
"import", "export", "from", "default", "try", "catch", "finally",
"throw", "true", "false", "null", "undefined", "NaN", "Infinity",
/// Paints `js_highlight` spans onto isocline's cell attributes.
const IcSink = struct {
henv: ?*c.ic_highlight_env_t,
pub fn emit(self: IcSink, start: usize, len: usize, kind: js_highlight.Kind) void {
const style: []const u8 = switch (kind) {
.comment => style_comment,
.string => style_string,
.variable => style_var,
.number => style_num,
.keyword => style_keyword,
.global => style_jsglobal,
};
c.ic_highlight(self.henv, @intCast(start), @intCast(len), style.ptr);
}
};
// Globals available in the JS-mode page context; highlighted so it's visible
// at the prompt that they're in scope.
const js_globals = [_][]const u8{ "document", "window", "globalThis", "console", "lp" };
fn isJsKeyword(tok: []const u8) bool {
for (js_keywords) |kw| {
if (std.mem.eql(u8, kw, tok)) return true;
}
return false;
}
fn isJsGlobal(tok: []const u8) bool {
for (js_globals) |g| {
if (std.mem.eql(u8, g, tok)) return true;
}
return false;
}
fn isIdChar(ch: u8) bool {
return std.ascii.isAlphanumeric(ch) or ch == '_' or ch == '$' or ch >= 0x80;
}
/// Highlight the buffer as JavaScript: keywords, strings (incl. template
/// literals), numbers, comments, and `$LP_*` env-var refs. Byte offsets are
/// safe (see `highlighterCallback`): every token boundary is an ASCII byte and
/// non-ASCII bytes advance singly without being highlighted.
/// Highlight the buffer as JavaScript. Byte offsets are safe (see
/// `highlighterCallback`): every token boundary is an ASCII byte and non-ASCII
/// bytes advance singly without being highlighted.
fn highlightJavaScript(henv: ?*c.ic_highlight_env_t, text: []const u8) void {
var i: usize = 0;
while (i < text.len) {
const ch = text[i];
if (ch == '/' and i + 1 < text.len and (text[i + 1] == '/' or text[i + 1] == '*')) {
const start = i;
if (text[i + 1] == '/') {
i = std.mem.indexOfScalarPos(u8, text, i + 2, '\n') orelse text.len;
} else {
const close = std.mem.indexOfPos(u8, text, i + 2, "*/");
i = if (close) |p| p + 2 else text.len;
}
c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_comment.ptr);
continue;
}
if (ch == '\'' or ch == '"' or ch == '`') {
const start = i;
i = scanQuoted(text, i);
c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_string.ptr);
// `$LP_*` repaints yellow over green: isocline merges per cell, so the later call wins.
highlightDollarVarsIn(henv, text, start, i);
continue;
}
if (ch == '$') {
const start = i;
i += 1;
while (i < text.len and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1;
if (i > start + 1) {
c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_var.ptr);
}
continue;
}
if (std.ascii.isDigit(ch) or (ch == '.' and i + 1 < text.len and std.ascii.isDigit(text[i + 1]))) {
const start = i;
i += 1;
while (i < text.len and (std.ascii.isHex(text[i]) or text[i] == '.' or text[i] == '_' or text[i] == 'x' or text[i] == 'X')) i += 1;
c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_num.ptr);
continue;
}
if (std.ascii.isAlphabetic(ch) or ch == '_') {
const start = i;
i += 1;
while (i < text.len and isIdChar(text[i])) i += 1;
const tok = text[start..i];
if (isJsKeyword(tok)) {
c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_keyword.ptr);
} else if (isJsGlobal(tok) and (start == 0 or text[start - 1] != '.')) {
// `.document` is a property access, not the global
c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_jsglobal.ptr);
}
continue;
}
i += 1;
}
const sink: IcSink = .{ .henv = henv };
_ = js_highlight.tokenize(text, .normal, sink);
}
fn highlightSlashArgs(henv: ?*c.ic_highlight_env_t, text: []const u8, start: usize) void {

243
src/agent/js_highlight.zig Normal file
View File

@@ -0,0 +1,243 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
pub const Kind = enum { comment, string, variable, number, keyword, global };
/// Carried between calls because fenced code is tokenized one line at a time:
/// block comments and template literals outlive a line boundary.
pub const State = enum { normal, block_comment, template };
pub const StringSpan = struct { end: usize, closed: bool };
/// Scan the quoted run opening at `text[start]`, including the triple-delimiter
/// form. Escapes are not honored, matching the prompt highlighter's behavior.
pub fn scanString(text: []const u8, start: usize) StringSpan {
if (start >= text.len) return .{ .end = start, .closed = false };
const ch = text[start];
if (start + 2 < text.len and text[start + 1] == ch and text[start + 2] == ch) {
const delim = text[start .. start + 3];
const close = std.mem.indexOfPos(u8, text, start + 3, delim) orelse
return .{ .end = text.len, .closed = false };
return .{ .end = close + 3, .closed = true };
}
const close = std.mem.indexOfScalarPos(u8, text, start + 1, ch) orelse
return .{ .end = text.len, .closed = false };
return .{ .end = close + 1, .closed = true };
}
/// Tokenize `text` as JavaScript starting from `state`, reporting spans to
/// `sink.emit(start, len, kind)` and returning the state at end of input.
/// Spans arrive in order and never overlap — the gaps between them are plain
/// text — so an emitter that writes sequentially (ANSI) works as well as one
/// that paints cells (isocline).
pub fn tokenize(text: []const u8, state: State, sink: anytype) State {
var i: usize = 0;
switch (state) {
.block_comment => {
const close = std.mem.indexOfPos(u8, text, 0, "*/");
i = if (close) |p| p + 2 else text.len;
if (i > 0) sink.emit(0, i, .comment);
if (close == null) return .block_comment;
},
.template => {
const close = std.mem.indexOfScalarPos(u8, text, 0, '`');
i = if (close) |p| p + 1 else text.len;
emitString(text, 0, i, sink);
if (close == null) return .template;
},
.normal => {},
}
while (i < text.len) {
const ch = text[i];
if (ch == '/' and i + 1 < text.len and (text[i + 1] == '/' or text[i + 1] == '*')) {
const start = i;
if (text[i + 1] == '/') {
i = std.mem.indexOfScalarPos(u8, text, i + 2, '\n') orelse text.len;
sink.emit(start, i - start, .comment);
continue;
}
const close = std.mem.indexOfPos(u8, text, i + 2, "*/");
i = if (close) |p| p + 2 else text.len;
sink.emit(start, i - start, .comment);
if (close == null) return .block_comment;
continue;
}
if (ch == '\'' or ch == '"' or ch == '`') {
const span = scanString(text, i);
emitString(text, i, span.end, sink);
i = span.end;
// Only a template literal may legally continue on the next line.
if (!span.closed and ch == '`') return .template;
continue;
}
if (ch == '$') {
const start = i;
i += 1;
while (i < text.len and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1;
if (i > start + 1) sink.emit(start, i - start, .variable);
continue;
}
if (std.ascii.isDigit(ch) or (ch == '.' and i + 1 < text.len and std.ascii.isDigit(text[i + 1]))) {
const start = i;
i += 1;
while (i < text.len and (std.ascii.isHex(text[i]) or text[i] == '.' or text[i] == '_' or text[i] == 'x' or text[i] == 'X')) i += 1;
sink.emit(start, i - start, .number);
continue;
}
if (std.ascii.isAlphabetic(ch) or ch == '_') {
const start = i;
i += 1;
while (i < text.len and isIdChar(text[i])) i += 1;
const tok = text[start..i];
if (isKeyword(tok)) {
sink.emit(start, i - start, .keyword);
} else if (isGlobal(tok) and (start == 0 or text[start - 1] != '.')) {
// `.document` is a property access, not the global
sink.emit(start, i - start, .global);
}
continue;
}
i += 1;
}
return .normal;
}
/// Emit `text[start..end]` as string spans split around any `$name` refs, so
/// the two never overlap. The prompt highlighter used to paint the ref over the
/// string and lean on isocline's last-write-wins per cell; splitting keeps the
/// same result while staying safe for a sequential writer.
fn emitString(text: []const u8, start: usize, end: usize, sink: anytype) void {
var seg = start;
var i = start;
while (i < end) {
if (text[i] != '$') {
i += 1;
continue;
}
const tok = i;
i += 1;
while (i < end and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1;
if (i > tok + 1) {
if (tok > seg) sink.emit(seg, tok - seg, .string);
sink.emit(tok, i - tok, .variable);
seg = i;
}
}
if (end > seg) sink.emit(seg, end - seg, .string);
}
const keywords = [_][]const u8{
"function", "async", "await", "yield", "return", "if", "else",
"for", "while", "do", "switch", "case", "break", "continue",
"var", "let", "const", "new", "delete", "typeof", "instanceof",
"in", "of", "void", "this", "super", "class", "extends",
"import", "export", "from", "default", "try", "catch", "finally",
"throw", "true", "false", "null", "undefined", "NaN", "Infinity",
};
// Globals available in the JS-mode page context; highlighted so it's visible
// at the prompt that they're in scope.
const globals = [_][]const u8{ "document", "window", "globalThis", "console", "lp" };
pub fn isKeyword(tok: []const u8) bool {
for (keywords) |kw| {
if (std.mem.eql(u8, kw, tok)) return true;
}
return false;
}
pub fn isGlobal(tok: []const u8) bool {
for (globals) |g| {
if (std.mem.eql(u8, g, tok)) return true;
}
return false;
}
pub fn isIdChar(ch: u8) bool {
return std.ascii.isAlphanumeric(ch) or ch == '_' or ch == '$' or ch >= 0x80;
}
const testing = std.testing;
/// Records spans as `kind:text` so tests read as the tokenization, not offsets.
const TestSink = struct {
text: []const u8,
buf: std.ArrayListUnmanaged(u8) = .empty,
last_end: usize = 0,
overlapped: bool = false,
fn emit(self: *TestSink, start: usize, len: usize, kind: Kind) void {
if (start < self.last_end) self.overlapped = true;
self.last_end = start + len;
self.buf.writer(testing.allocator).print("{s}:{s} ", .{
@tagName(kind), self.text[start..][0..len],
}) catch unreachable;
}
};
fn expectTokens(expected: []const u8, src: []const u8) !void {
var sink: TestSink = .{ .text = src };
defer sink.buf.deinit(testing.allocator);
_ = tokenize(src, .normal, &sink);
try testing.expect(!sink.overlapped);
try testing.expectEqualStrings(expected, std.mem.trimRight(u8, sink.buf.items, " "));
}
test "js_highlight: keywords, globals, numbers" {
try expectTokens("keyword:const number:1", "const x = 1");
try expectTokens("global:document", "document.body");
// A property access is not the global.
try expectTokens("", "el.document");
}
test "js_highlight: comments and strings" {
try expectTokens("comment:// hi", "// hi");
try expectTokens("comment:/* hi */ keyword:const", "/* hi */ const");
try expectTokens("string:'hi'", "'hi'");
}
test "js_highlight: $refs inside strings do not overlap" {
try expectTokens("string:'a variable:$LP_KEY string:'", "'a $LP_KEY'");
try expectTokens("variable:$LP_KEY", "$LP_KEY");
}
test "js_highlight: block comment spans lines" {
var sink: TestSink = .{ .text = "/* open" };
defer sink.buf.deinit(testing.allocator);
try testing.expectEqual(State.block_comment, tokenize("/* open", .normal, &sink));
var sink2: TestSink = .{ .text = "still */ const" };
defer sink2.buf.deinit(testing.allocator);
try testing.expectEqual(State.normal, tokenize("still */ const", .block_comment, &sink2));
try testing.expectEqualStrings("comment:still */ keyword:const", std.mem.trimRight(u8, sink2.buf.items, " "));
}
test "js_highlight: template literal spans lines" {
var sink: TestSink = .{ .text = "`<div>" };
defer sink.buf.deinit(testing.allocator);
try testing.expectEqual(State.template, tokenize("`<div>", .normal, &sink));
var sink2: TestSink = .{ .text = "</div>` + x" };
defer sink2.buf.deinit(testing.allocator);
try testing.expectEqual(State.normal, tokenize("</div>` + x", .template, &sink2));
try testing.expectEqualStrings("string:</div>`", std.mem.trimRight(u8, sink2.buf.items, " "));
}

94
src/agent/md_term.zig
View File

@@ -17,6 +17,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const js_highlight = @import("js_highlight.zig");
pub const ansi = struct {
pub const reset = "\x1b[0m";
@@ -29,6 +30,8 @@ pub const ansi = struct {
pub const green = "\x1b[32m";
pub const yellow = "\x1b[33m";
pub const red = "\x1b[31m";
pub const blue = "\x1b[34m";
pub const magenta = "\x1b[35m";
pub const clear_eol = "\x1b[K";
pub const clear_line = "\x1b[2K";
};
@@ -36,12 +39,14 @@ pub const ansi = struct {
/// Render markdown `src` as ANSI-styled terminal output to `w`.
pub fn render(w: *std.Io.Writer, src: []const u8) !void {
var in_fence = false;
var js: js_highlight.State = .normal;
var wrote_any = false;
var it = std.mem.splitScalar(u8, src, '\n');
while (it.next()) |line| {
// Drop the ``` delimiter entirely: no line, no separator.
if (isFenceDelimiter(line)) {
in_fence = !in_fence;
js = .normal;
continue;
}
if (wrote_any) try w.writeByte('\n');
@@ -52,7 +57,7 @@ pub fn render(w: *std.Io.Writer, src: []const u8) !void {
continue;
};
}
try renderLine(w, line, in_fence);
try renderLine(w, line, if (in_fence) &js else null);
}
}
@@ -273,6 +278,8 @@ fn isTableSeparator(line: []const u8) bool {
pub const Stream = struct {
len: usize = 0,
in_fence: bool = false,
/// Fenced-code lexer state, carried across lines and chunks.
js_state: js_highlight.State = .normal,
/// A line that outgrew `buf` passes through unrendered to its newline.
raw: bool = false,
/// `held`: one pipe row buffered, pending the separator that confirms a
@@ -343,11 +350,13 @@ pub const Stream = struct {
}
const partial = self.buf[0..self.len];
const in_fence = self.in_fence;
var js = self.js_state;
self.len = 0;
self.raw = false;
self.in_fence = false;
self.js_state = .normal;
if (partial.len == 0 or isFenceDelimiter(partial)) return;
try renderLine(w, partial, in_fence);
try renderLine(w, partial, if (in_fence) &js else null);
}
fn emitLine(self: *Stream, w: *std.Io.Writer, text: []const u8) std.Io.Writer.Error!void {
@@ -355,13 +364,14 @@ pub const Stream = struct {
.text => {
if (isFenceDelimiter(text)) {
self.in_fence = !self.in_fence;
self.js_state = .normal;
return;
}
if (!self.in_fence and isTableRow(text) and self.appendTableLine(text)) {
self.mode = .held;
return;
}
try renderLine(w, text, self.in_fence);
try renderLine(w, text, if (self.in_fence) &self.js_state else null);
try w.writeByte('\n');
},
.held => {
@@ -379,7 +389,7 @@ pub const Stream = struct {
if (isTableRow(text)) {
if (self.appendTableLine(text)) return;
try self.flushTableUnaligned(w);
try renderLine(w, text, false);
try renderLine(w, text, null);
try w.writeByte('\n');
return;
}
@@ -391,7 +401,7 @@ pub const Stream = struct {
/// No separator followed, so the held row wasn't a table header.
fn releaseHeldRow(self: *Stream, w: *std.Io.Writer) std.Io.Writer.Error!void {
try renderLine(w, self.tableText(), false);
try renderLine(w, self.tableText(), null);
try w.writeByte('\n');
self.resetTable();
}
@@ -409,7 +419,7 @@ pub const Stream = struct {
try w.writeAll(clear_placeholder);
var it = std.mem.splitScalar(u8, self.tableText(), '\n');
while (it.next()) |line| {
try renderLine(w, line, false);
try renderLine(w, line, null);
try w.writeByte('\n');
}
self.resetTable();
@@ -437,9 +447,10 @@ pub const Stream = struct {
}
};
fn renderLine(w: *std.Io.Writer, line: []const u8, in_fence: bool) !void {
if (in_fence) {
try styled(w, line, ansi.cyan);
/// `js` carries the fenced-code lexer state across lines; null outside a fence.
fn renderLine(w: *std.Io.Writer, line: []const u8, js: ?*js_highlight.State) !void {
if (js) |state| {
state.* = try renderCodeLine(w, line, state.*);
return;
}
@@ -571,6 +582,53 @@ fn span(w: *std.Io.Writer, inner: []const u8, style: []const u8, active: ?*const
try Style.applyOpt(active, w);
}
/// Writes `js_highlight` spans as ANSI, filling the gaps between them with
/// unstyled text. `emit` cannot fail, so a write error is stashed and returned
/// by `finish`.
const JsSink = struct {
w: *std.Io.Writer,
text: []const u8,
last: usize = 0,
err: ?std.Io.Writer.Error = null,
fn color(kind: js_highlight.Kind) []const u8 {
return switch (kind) {
.comment => ansi.dim ++ ansi.italic,
.string => ansi.green,
.variable => ansi.yellow ++ ansi.bold,
.number => ansi.magenta,
.keyword => ansi.blue ++ ansi.bold,
.global => ansi.cyan,
};
}
pub fn emit(self: *JsSink, start: usize, len: usize, kind: js_highlight.Kind) void {
self.write(start, len, kind) catch |err| {
self.err = self.err orelse err;
};
}
fn write(self: *JsSink, start: usize, len: usize, kind: js_highlight.Kind) !void {
if (start > self.last) try self.w.writeAll(self.text[self.last..start]);
try styled(self.w, self.text[start..][0..len], color(kind));
self.last = start + len;
}
fn finish(self: *JsSink) !void {
if (self.err) |err| return err;
if (self.last < self.text.len) try self.w.writeAll(self.text[self.last..]);
}
};
/// Syntax-highlight one line of fenced code as JavaScript — the only language
/// this agent emits in practice. Returns the lexer state for the next line.
fn renderCodeLine(w: *std.Io.Writer, line: []const u8, state: js_highlight.State) !js_highlight.State {
var sink: JsSink = .{ .w = w, .text = line };
const next = js_highlight.tokenize(line, state, &sink);
try sink.finish();
return next;
}
fn styled(w: *std.Io.Writer, inner: []const u8, style: []const u8) !void {
try w.writeAll(style);
try w.writeAll(inner);
@@ -661,8 +719,20 @@ test "md_term: nested inline styles" {
);
}
test "md_term: fenced code block" {
try expectRender("\x1b[36mlet x = 1;\x1b[0m", "```\nlet x = 1;\n```");
test "md_term: fenced code block is highlighted as JavaScript" {
try expectRender(
"\x1b[34m\x1b[1mlet\x1b[0m x = \x1b[35m1\x1b[0m;",
"```\nlet x = 1;\n```",
);
// Untokenized text passes through unstyled.
try expectRender("plain", "```\nplain\n```");
}
test "md_term: fenced template literal spans lines" {
try expectRender(
"\x1b[32m`<div>\x1b[0m\n\x1b[32m</div>`\x1b[0m",
"```\n`<div>\n</div>`\n```",
);
}
test "md_term: link" {
@@ -798,7 +868,7 @@ test "md_term: stream fence state spans chunks" {
try s.feed(&aw.writer, "`\nafter\n");
try s.close(&aw.writer);
try testing.expectEqualStrings(
"\x1b[36mcode\x1b[0m\nafter\n",
"code\nafter\n",
aw.written(),
);
}