From 8b1686f6fb00a5c80f6b3903f1b6ba7b5e69a591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 15 Jul 2026 09:10:11 +0200 Subject: [PATCH] agent: highlight markdown code blocks as JS Extracts the JavaScript highlighting logic from `Terminal.zig` into a reusable `js_highlight.zig` module. Uses this module to syntax-highlight markdown fenced code blocks in `md_term.zig` using ANSI escape codes. --- src/agent/Terminal.zig | 121 ++++-------------- src/agent/js_highlight.zig | 243 +++++++++++++++++++++++++++++++++++++ src/agent/md_term.zig | 94 ++++++++++++-- 3 files changed, 348 insertions(+), 110 deletions(-) create mode 100644 src/agent/js_highlight.zig diff --git a/src/agent/Terminal.zig b/src/agent/Terminal.zig index 397fdef2b..57f05aa46 100644 --- a/src/agent/Terminal.zig +++ b/src/agent/Terminal.zig @@ -25,6 +25,7 @@ const Schema = lp.Schema; const SlashCommand = @import("SlashCommand.zig"); const Spinner = @import("Spinner.zig"); const md_term = @import("md_term.zig"); +const js_highlight = @import("js_highlight.zig"); const c = @cImport({ @cInclude("isocline.h"); }); @@ -813,19 +814,9 @@ fn highlightBareToken(henv: ?*c.ic_highlight_env_t, text: []const u8, start: usi } else |_| {} } -/// Returns the index just past the matching closing quote, or `text.len` if -/// unterminated. Does not handle backslash escapes (matches Schema.tokenize). +/// Index just past the matching closing quote, or `text.len` if unterminated. fn scanQuoted(text: []const u8, start: usize) usize { - if (start >= text.len) return start; - const ch = text[start]; - const is_triple = start + 2 < text.len and text[start + 1] == ch and text[start + 2] == ch; - if (is_triple) { - const triple_delim = text[start .. start + 3]; - const close = std.mem.indexOfPos(u8, text, start + 3, triple_delim) orelse return text.len; - return close + 3; - } - const close = std.mem.indexOfScalarPos(u8, text, start + 1, ch) orelse return text.len; - return close + 1; + return js_highlight.scanString(text, start).end; } /// Highlight `$LP_*` tokens appearing from `start` onward. @@ -853,95 +844,29 @@ fn highlightDollarVarsIn(henv: ?*c.ic_highlight_env_t, text: []const u8, start: } } -const js_keywords = [_][]const u8{ - "function", "async", "await", "yield", "return", "if", "else", - "for", "while", "do", "switch", "case", "break", "continue", - "var", "let", "const", "new", "delete", "typeof", "instanceof", - "in", "of", "void", "this", "super", "class", "extends", - "import", "export", "from", "default", "try", "catch", "finally", - "throw", "true", "false", "null", "undefined", "NaN", "Infinity", +/// Paints `js_highlight` spans onto isocline's cell attributes. +const IcSink = struct { + henv: ?*c.ic_highlight_env_t, + + pub fn emit(self: IcSink, start: usize, len: usize, kind: js_highlight.Kind) void { + const style: []const u8 = switch (kind) { + .comment => style_comment, + .string => style_string, + .variable => style_var, + .number => style_num, + .keyword => style_keyword, + .global => style_jsglobal, + }; + c.ic_highlight(self.henv, @intCast(start), @intCast(len), style.ptr); + } }; -// Globals available in the JS-mode page context; highlighted so it's visible -// at the prompt that they're in scope. -const js_globals = [_][]const u8{ "document", "window", "globalThis", "console", "lp" }; - -fn isJsKeyword(tok: []const u8) bool { - for (js_keywords) |kw| { - if (std.mem.eql(u8, kw, tok)) return true; - } - return false; -} - -fn isJsGlobal(tok: []const u8) bool { - for (js_globals) |g| { - if (std.mem.eql(u8, g, tok)) return true; - } - return false; -} - -fn isIdChar(ch: u8) bool { - return std.ascii.isAlphanumeric(ch) or ch == '_' or ch == '$' or ch >= 0x80; -} - -/// Highlight the buffer as JavaScript: keywords, strings (incl. template -/// literals), numbers, comments, and `$LP_*` env-var refs. Byte offsets are -/// safe (see `highlighterCallback`): every token boundary is an ASCII byte and -/// non-ASCII bytes advance singly without being highlighted. +/// Highlight the buffer as JavaScript. Byte offsets are safe (see +/// `highlighterCallback`): every token boundary is an ASCII byte and non-ASCII +/// bytes advance singly without being highlighted. fn highlightJavaScript(henv: ?*c.ic_highlight_env_t, text: []const u8) void { - var i: usize = 0; - while (i < text.len) { - const ch = text[i]; - if (ch == '/' and i + 1 < text.len and (text[i + 1] == '/' or text[i + 1] == '*')) { - const start = i; - if (text[i + 1] == '/') { - i = std.mem.indexOfScalarPos(u8, text, i + 2, '\n') orelse text.len; - } else { - const close = std.mem.indexOfPos(u8, text, i + 2, "*/"); - i = if (close) |p| p + 2 else text.len; - } - c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_comment.ptr); - continue; - } - if (ch == '\'' or ch == '"' or ch == '`') { - const start = i; - i = scanQuoted(text, i); - c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_string.ptr); - // `$LP_*` repaints yellow over green: isocline merges per cell, so the later call wins. - highlightDollarVarsIn(henv, text, start, i); - continue; - } - if (ch == '$') { - const start = i; - i += 1; - while (i < text.len and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1; - if (i > start + 1) { - c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_var.ptr); - } - continue; - } - if (std.ascii.isDigit(ch) or (ch == '.' and i + 1 < text.len and std.ascii.isDigit(text[i + 1]))) { - const start = i; - i += 1; - while (i < text.len and (std.ascii.isHex(text[i]) or text[i] == '.' or text[i] == '_' or text[i] == 'x' or text[i] == 'X')) i += 1; - c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_num.ptr); - continue; - } - if (std.ascii.isAlphabetic(ch) or ch == '_') { - const start = i; - i += 1; - while (i < text.len and isIdChar(text[i])) i += 1; - const tok = text[start..i]; - if (isJsKeyword(tok)) { - c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_keyword.ptr); - } else if (isJsGlobal(tok) and (start == 0 or text[start - 1] != '.')) { - // `.document` is a property access, not the global - c.ic_highlight(henv, @intCast(start), @intCast(i - start), style_jsglobal.ptr); - } - continue; - } - i += 1; - } + const sink: IcSink = .{ .henv = henv }; + _ = js_highlight.tokenize(text, .normal, sink); } fn highlightSlashArgs(henv: ?*c.ic_highlight_env_t, text: []const u8, start: usize) void { diff --git a/src/agent/js_highlight.zig b/src/agent/js_highlight.zig new file mode 100644 index 000000000..630fb16d8 --- /dev/null +++ b/src/agent/js_highlight.zig @@ -0,0 +1,243 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); + +pub const Kind = enum { comment, string, variable, number, keyword, global }; + +/// Carried between calls because fenced code is tokenized one line at a time: +/// block comments and template literals outlive a line boundary. +pub const State = enum { normal, block_comment, template }; + +pub const StringSpan = struct { end: usize, closed: bool }; + +/// Scan the quoted run opening at `text[start]`, including the triple-delimiter +/// form. Escapes are not honored, matching the prompt highlighter's behavior. +pub fn scanString(text: []const u8, start: usize) StringSpan { + if (start >= text.len) return .{ .end = start, .closed = false }; + const ch = text[start]; + if (start + 2 < text.len and text[start + 1] == ch and text[start + 2] == ch) { + const delim = text[start .. start + 3]; + const close = std.mem.indexOfPos(u8, text, start + 3, delim) orelse + return .{ .end = text.len, .closed = false }; + return .{ .end = close + 3, .closed = true }; + } + const close = std.mem.indexOfScalarPos(u8, text, start + 1, ch) orelse + return .{ .end = text.len, .closed = false }; + return .{ .end = close + 1, .closed = true }; +} + +/// Tokenize `text` as JavaScript starting from `state`, reporting spans to +/// `sink.emit(start, len, kind)` and returning the state at end of input. +/// Spans arrive in order and never overlap — the gaps between them are plain +/// text — so an emitter that writes sequentially (ANSI) works as well as one +/// that paints cells (isocline). +pub fn tokenize(text: []const u8, state: State, sink: anytype) State { + var i: usize = 0; + + switch (state) { + .block_comment => { + const close = std.mem.indexOfPos(u8, text, 0, "*/"); + i = if (close) |p| p + 2 else text.len; + if (i > 0) sink.emit(0, i, .comment); + if (close == null) return .block_comment; + }, + .template => { + const close = std.mem.indexOfScalarPos(u8, text, 0, '`'); + i = if (close) |p| p + 1 else text.len; + emitString(text, 0, i, sink); + if (close == null) return .template; + }, + .normal => {}, + } + + while (i < text.len) { + const ch = text[i]; + if (ch == '/' and i + 1 < text.len and (text[i + 1] == '/' or text[i + 1] == '*')) { + const start = i; + if (text[i + 1] == '/') { + i = std.mem.indexOfScalarPos(u8, text, i + 2, '\n') orelse text.len; + sink.emit(start, i - start, .comment); + continue; + } + const close = std.mem.indexOfPos(u8, text, i + 2, "*/"); + i = if (close) |p| p + 2 else text.len; + sink.emit(start, i - start, .comment); + if (close == null) return .block_comment; + continue; + } + if (ch == '\'' or ch == '"' or ch == '`') { + const span = scanString(text, i); + emitString(text, i, span.end, sink); + i = span.end; + // Only a template literal may legally continue on the next line. + if (!span.closed and ch == '`') return .template; + continue; + } + if (ch == '$') { + const start = i; + i += 1; + while (i < text.len and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1; + if (i > start + 1) sink.emit(start, i - start, .variable); + continue; + } + if (std.ascii.isDigit(ch) or (ch == '.' and i + 1 < text.len and std.ascii.isDigit(text[i + 1]))) { + const start = i; + i += 1; + while (i < text.len and (std.ascii.isHex(text[i]) or text[i] == '.' or text[i] == '_' or text[i] == 'x' or text[i] == 'X')) i += 1; + sink.emit(start, i - start, .number); + continue; + } + if (std.ascii.isAlphabetic(ch) or ch == '_') { + const start = i; + i += 1; + while (i < text.len and isIdChar(text[i])) i += 1; + const tok = text[start..i]; + if (isKeyword(tok)) { + sink.emit(start, i - start, .keyword); + } else if (isGlobal(tok) and (start == 0 or text[start - 1] != '.')) { + // `.document` is a property access, not the global + sink.emit(start, i - start, .global); + } + continue; + } + i += 1; + } + return .normal; +} + +/// Emit `text[start..end]` as string spans split around any `$name` refs, so +/// the two never overlap. The prompt highlighter used to paint the ref over the +/// string and lean on isocline's last-write-wins per cell; splitting keeps the +/// same result while staying safe for a sequential writer. +fn emitString(text: []const u8, start: usize, end: usize, sink: anytype) void { + var seg = start; + var i = start; + while (i < end) { + if (text[i] != '$') { + i += 1; + continue; + } + const tok = i; + i += 1; + while (i < end and (std.ascii.isAlphanumeric(text[i]) or text[i] == '_')) i += 1; + if (i > tok + 1) { + if (tok > seg) sink.emit(seg, tok - seg, .string); + sink.emit(tok, i - tok, .variable); + seg = i; + } + } + if (end > seg) sink.emit(seg, end - seg, .string); +} + +const keywords = [_][]const u8{ + "function", "async", "await", "yield", "return", "if", "else", + "for", "while", "do", "switch", "case", "break", "continue", + "var", "let", "const", "new", "delete", "typeof", "instanceof", + "in", "of", "void", "this", "super", "class", "extends", + "import", "export", "from", "default", "try", "catch", "finally", + "throw", "true", "false", "null", "undefined", "NaN", "Infinity", +}; + +// Globals available in the JS-mode page context; highlighted so it's visible +// at the prompt that they're in scope. +const globals = [_][]const u8{ "document", "window", "globalThis", "console", "lp" }; + +pub fn isKeyword(tok: []const u8) bool { + for (keywords) |kw| { + if (std.mem.eql(u8, kw, tok)) return true; + } + return false; +} + +pub fn isGlobal(tok: []const u8) bool { + for (globals) |g| { + if (std.mem.eql(u8, g, tok)) return true; + } + return false; +} + +pub fn isIdChar(ch: u8) bool { + return std.ascii.isAlphanumeric(ch) or ch == '_' or ch == '$' or ch >= 0x80; +} + +const testing = std.testing; + +/// Records spans as `kind:text` so tests read as the tokenization, not offsets. +const TestSink = struct { + text: []const u8, + buf: std.ArrayListUnmanaged(u8) = .empty, + last_end: usize = 0, + overlapped: bool = false, + + fn emit(self: *TestSink, start: usize, len: usize, kind: Kind) void { + if (start < self.last_end) self.overlapped = true; + self.last_end = start + len; + self.buf.writer(testing.allocator).print("{s}:{s} ", .{ + @tagName(kind), self.text[start..][0..len], + }) catch unreachable; + } +}; + +fn expectTokens(expected: []const u8, src: []const u8) !void { + var sink: TestSink = .{ .text = src }; + defer sink.buf.deinit(testing.allocator); + _ = tokenize(src, .normal, &sink); + try testing.expect(!sink.overlapped); + try testing.expectEqualStrings(expected, std.mem.trimRight(u8, sink.buf.items, " ")); +} + +test "js_highlight: keywords, globals, numbers" { + try expectTokens("keyword:const number:1", "const x = 1"); + try expectTokens("global:document", "document.body"); + // A property access is not the global. + try expectTokens("", "el.document"); +} + +test "js_highlight: comments and strings" { + try expectTokens("comment:// hi", "// hi"); + try expectTokens("comment:/* hi */ keyword:const", "/* hi */ const"); + try expectTokens("string:'hi'", "'hi'"); +} + +test "js_highlight: $refs inside strings do not overlap" { + try expectTokens("string:'a variable:$LP_KEY string:'", "'a $LP_KEY'"); + try expectTokens("variable:$LP_KEY", "$LP_KEY"); +} + +test "js_highlight: block comment spans lines" { + var sink: TestSink = .{ .text = "/* open" }; + defer sink.buf.deinit(testing.allocator); + try testing.expectEqual(State.block_comment, tokenize("/* open", .normal, &sink)); + + var sink2: TestSink = .{ .text = "still */ const" }; + defer sink2.buf.deinit(testing.allocator); + try testing.expectEqual(State.normal, tokenize("still */ const", .block_comment, &sink2)); + try testing.expectEqualStrings("comment:still */ keyword:const", std.mem.trimRight(u8, sink2.buf.items, " ")); +} + +test "js_highlight: template literal spans lines" { + var sink: TestSink = .{ .text = "`
" }; + defer sink.buf.deinit(testing.allocator); + try testing.expectEqual(State.template, tokenize("`
", .normal, &sink)); + + var sink2: TestSink = .{ .text = "
` + x" }; + defer sink2.buf.deinit(testing.allocator); + try testing.expectEqual(State.normal, tokenize("
` + x", .template, &sink2)); + try testing.expectEqualStrings("string:`", std.mem.trimRight(u8, sink2.buf.items, " ")); +} diff --git a/src/agent/md_term.zig b/src/agent/md_term.zig index 864d891d2..d9b697a53 100644 --- a/src/agent/md_term.zig +++ b/src/agent/md_term.zig @@ -17,6 +17,7 @@ // along with this program. If not, see . const std = @import("std"); +const js_highlight = @import("js_highlight.zig"); pub const ansi = struct { pub const reset = "\x1b[0m"; @@ -29,6 +30,8 @@ pub const ansi = struct { pub const green = "\x1b[32m"; pub const yellow = "\x1b[33m"; pub const red = "\x1b[31m"; + pub const blue = "\x1b[34m"; + pub const magenta = "\x1b[35m"; pub const clear_eol = "\x1b[K"; pub const clear_line = "\x1b[2K"; }; @@ -36,12 +39,14 @@ pub const ansi = struct { /// Render markdown `src` as ANSI-styled terminal output to `w`. pub fn render(w: *std.Io.Writer, src: []const u8) !void { var in_fence = false; + var js: js_highlight.State = .normal; var wrote_any = false; var it = std.mem.splitScalar(u8, src, '\n'); while (it.next()) |line| { // Drop the ``` delimiter entirely: no line, no separator. if (isFenceDelimiter(line)) { in_fence = !in_fence; + js = .normal; continue; } if (wrote_any) try w.writeByte('\n'); @@ -52,7 +57,7 @@ pub fn render(w: *std.Io.Writer, src: []const u8) !void { continue; }; } - try renderLine(w, line, in_fence); + try renderLine(w, line, if (in_fence) &js else null); } } @@ -273,6 +278,8 @@ fn isTableSeparator(line: []const u8) bool { pub const Stream = struct { len: usize = 0, in_fence: bool = false, + /// Fenced-code lexer state, carried across lines and chunks. + js_state: js_highlight.State = .normal, /// A line that outgrew `buf` passes through unrendered to its newline. raw: bool = false, /// `held`: one pipe row buffered, pending the separator that confirms a @@ -343,11 +350,13 @@ pub const Stream = struct { } const partial = self.buf[0..self.len]; const in_fence = self.in_fence; + var js = self.js_state; self.len = 0; self.raw = false; self.in_fence = false; + self.js_state = .normal; if (partial.len == 0 or isFenceDelimiter(partial)) return; - try renderLine(w, partial, in_fence); + try renderLine(w, partial, if (in_fence) &js else null); } fn emitLine(self: *Stream, w: *std.Io.Writer, text: []const u8) std.Io.Writer.Error!void { @@ -355,13 +364,14 @@ pub const Stream = struct { .text => { if (isFenceDelimiter(text)) { self.in_fence = !self.in_fence; + self.js_state = .normal; return; } if (!self.in_fence and isTableRow(text) and self.appendTableLine(text)) { self.mode = .held; return; } - try renderLine(w, text, self.in_fence); + try renderLine(w, text, if (self.in_fence) &self.js_state else null); try w.writeByte('\n'); }, .held => { @@ -379,7 +389,7 @@ pub const Stream = struct { if (isTableRow(text)) { if (self.appendTableLine(text)) return; try self.flushTableUnaligned(w); - try renderLine(w, text, false); + try renderLine(w, text, null); try w.writeByte('\n'); return; } @@ -391,7 +401,7 @@ pub const Stream = struct { /// No separator followed, so the held row wasn't a table header. fn releaseHeldRow(self: *Stream, w: *std.Io.Writer) std.Io.Writer.Error!void { - try renderLine(w, self.tableText(), false); + try renderLine(w, self.tableText(), null); try w.writeByte('\n'); self.resetTable(); } @@ -409,7 +419,7 @@ pub const Stream = struct { try w.writeAll(clear_placeholder); var it = std.mem.splitScalar(u8, self.tableText(), '\n'); while (it.next()) |line| { - try renderLine(w, line, false); + try renderLine(w, line, null); try w.writeByte('\n'); } self.resetTable(); @@ -437,9 +447,10 @@ pub const Stream = struct { } }; -fn renderLine(w: *std.Io.Writer, line: []const u8, in_fence: bool) !void { - if (in_fence) { - try styled(w, line, ansi.cyan); +/// `js` carries the fenced-code lexer state across lines; null outside a fence. +fn renderLine(w: *std.Io.Writer, line: []const u8, js: ?*js_highlight.State) !void { + if (js) |state| { + state.* = try renderCodeLine(w, line, state.*); return; } @@ -571,6 +582,53 @@ fn span(w: *std.Io.Writer, inner: []const u8, style: []const u8, active: ?*const try Style.applyOpt(active, w); } +/// Writes `js_highlight` spans as ANSI, filling the gaps between them with +/// unstyled text. `emit` cannot fail, so a write error is stashed and returned +/// by `finish`. +const JsSink = struct { + w: *std.Io.Writer, + text: []const u8, + last: usize = 0, + err: ?std.Io.Writer.Error = null, + + fn color(kind: js_highlight.Kind) []const u8 { + return switch (kind) { + .comment => ansi.dim ++ ansi.italic, + .string => ansi.green, + .variable => ansi.yellow ++ ansi.bold, + .number => ansi.magenta, + .keyword => ansi.blue ++ ansi.bold, + .global => ansi.cyan, + }; + } + + pub fn emit(self: *JsSink, start: usize, len: usize, kind: js_highlight.Kind) void { + self.write(start, len, kind) catch |err| { + self.err = self.err orelse err; + }; + } + + fn write(self: *JsSink, start: usize, len: usize, kind: js_highlight.Kind) !void { + if (start > self.last) try self.w.writeAll(self.text[self.last..start]); + try styled(self.w, self.text[start..][0..len], color(kind)); + self.last = start + len; + } + + fn finish(self: *JsSink) !void { + if (self.err) |err| return err; + if (self.last < self.text.len) try self.w.writeAll(self.text[self.last..]); + } +}; + +/// Syntax-highlight one line of fenced code as JavaScript — the only language +/// this agent emits in practice. Returns the lexer state for the next line. +fn renderCodeLine(w: *std.Io.Writer, line: []const u8, state: js_highlight.State) !js_highlight.State { + var sink: JsSink = .{ .w = w, .text = line }; + const next = js_highlight.tokenize(line, state, &sink); + try sink.finish(); + return next; +} + fn styled(w: *std.Io.Writer, inner: []const u8, style: []const u8) !void { try w.writeAll(style); try w.writeAll(inner); @@ -661,8 +719,20 @@ test "md_term: nested inline styles" { ); } -test "md_term: fenced code block" { - try expectRender("\x1b[36mlet x = 1;\x1b[0m", "```\nlet x = 1;\n```"); +test "md_term: fenced code block is highlighted as JavaScript" { + try expectRender( + "\x1b[34m\x1b[1mlet\x1b[0m x = \x1b[35m1\x1b[0m;", + "```\nlet x = 1;\n```", + ); + // Untokenized text passes through unstyled. + try expectRender("plain", "```\nplain\n```"); +} + +test "md_term: fenced template literal spans lines" { + try expectRender( + "\x1b[32m`
\x1b[0m\n\x1b[32m
`\x1b[0m", + "```\n`
\n
`\n```", + ); } test "md_term: link" { @@ -798,7 +868,7 @@ test "md_term: stream fence state spans chunks" { try s.feed(&aw.writer, "`\nafter\n"); try s.close(&aw.writer); try testing.expectEqualStrings( - "\x1b[36mcode\x1b[0m\nafter\n", + "code\nafter\n", aw.written(), ); }