mirror/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-06-11 09:35:59 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2196 from lightpanda-io/sqlite

Browse Source 
Sqlite
This commit is contained in:
Karl Seguin
2026-04-22 06:30:00 +08:00
committed by GitHub
parent bfb5ea1d94 e7be3c8b14
commit 650c0cbd28
10 changed files with 280896 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
build.zig
View File

@@ -91,6 +91,27 @@ pub fn build(b: *Build) !void {
break :blk mod;
};
lightpanda_module.addCSourceFile(.{
.file = b.path("lib/sqlite3/sqlite3.c"),
.flags = &[_][]const u8{
"-DSQLITE_DQS=0",
"-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1",
"-DSQLITE_USE_ALLOCA=1",
"-DSQLITE_THREADSAFE=1",
"-DSQLITE_TEMP_STORE=3",
"-DSQLITE_ENABLE_API_ARMOR=1",
"-DSQLITE_ENABLE_UNLOCK_NOTIFY",
"-DSQLITE_DEFAULT_FILE_PERMISSIONS=0600",
"-DSQLITE_OMIT_DECLTYPE=1",
"-DSQLITE_OMIT_DEPRECATED=1",
"-DSQLITE_OMIT_LOAD_EXTENSION=1",
"-DSQLITE_OMIT_PROGRESS_CALLBACK=1",
"-DSQLITE_OMIT_SHARED_CACHE",
"-DSQLITE_OMIT_TRACE=1",
"-DSQLITE_OMIT_UTF16=1",
},
});
// Check compilation
const check = b.step("check", "Check if lightpanda compiles");

265977
lib/sqlite3/sqlite3.c Normal file
View File

File diff suppressed because it is too large Load Diff

24
src/App.zig
View File

@@ -24,6 +24,7 @@ const Snapshot = @import("browser/js/Snapshot.zig");
const Platform = @import("browser/js/Platform.zig");
const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
const Storage = @import("storage/Storage.zig");
const Network = @import("network/Network.zig");
pub const ArenaPool = @import("ArenaPool.zig");
@@ -34,6 +35,7 @@ const App = @This();
network: Network,
config: *const Config,
storage: Storage,
platform: Platform,
snapshot: Snapshot,
telemetry: Telemetry,
@@ -42,26 +44,29 @@ arena_pool: ArenaPool,
app_dir_path: ?[]const u8,
pub fn init(allocator: Allocator, config: *const Config) !*App {
const platform = try Platform.init();
errdefer platform.deinit();
const snapshot = try Snapshot.load();
errdefer snapshot.deinit();
var storage = try Storage.init(allocator, config);
errdefer storage.deinit(allocator);
const app = try allocator.create(App);
errdefer allocator.destroy(app);
app.* = .{
.config = config,
.allocator = allocator,
.platform = platform,
.snapshot = snapshot,
.storage = storage,
.network = undefined,
.platform = undefined,
.snapshot = undefined,
.app_dir_path = undefined,
.telemetry = undefined,
.arena_pool = undefined,
};
app.platform = try Platform.init();
errdefer app.platform.deinit();
app.snapshot = try Snapshot.load();
errdefer app.snapshot.deinit();
app.network = try Network.init(allocator, app, config);
errdefer app.network.deinit();
@@ -91,6 +96,7 @@ pub fn deinit(self: *App) void {
self.snapshot.deinit();
self.platform.deinit();
self.arena_pool.deinit();
self.storage.deinit(allocator);
allocator.destroy(self);
}

46
src/Config.zig
View File

@@ -23,6 +23,7 @@ const builtin = @import("builtin");
const dump = @import("browser/dump.zig");
const mcp = @import("mcp.zig");
const Storage = @import("storage/Storage.zig");
const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;
const log = lp.log;
@@ -258,6 +259,20 @@ pub fn maxPendingConnections(self: *const Config) u31 {
};
}
pub fn storageEngine(self: *const Config) ?Storage.EngineType {
return switch (self.mode) {
inline .serve, .fetch, .mcp => |opts| opts.common.storage_engine,
else => unreachable,
};
}
pub fn storageSqlitePath(self: *const Config) ?[:0]const u8 {
return switch (self.mode) {
inline .serve, .fetch, .mcp => |opts| opts.common.storage_sqlite_path,
else => unreachable,
};
}
pub const Mode = union(RunMode) {
help: bool, // false when being printed because of an error
fetch: Fetch,
@@ -328,6 +343,8 @@ pub const Common = struct {
http_cache_dir: ?[]const u8 = null,
cookie: ?[]const u8 = null,
cookie_jar: ?[]const u8 = null,
storage_engine: ?Storage.EngineType = null,
storage_sqlite_path: ?[:0]const u8 = null,
web_bot_auth_key_file: ?[]const u8 = null,
web_bot_auth_keyid: ?[]const u8 = null,
@@ -483,6 +500,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
\\ Path to a directory to use as a Filesystem Cache for network resources.
\\ Omitting this will result is no caching.
\\ Defaults to no caching.
\\
\\--storage-engine
\\ The storage engine to use. Choices are: sqlite.
\\ Default to sqlite.
\\
\\--storage-sqlite-path
\\ Path to SQLite database file for persistent storage.
\\ Use ":memory:" for in-memory storage.
;
// MAX_HELP_LEN|
@@ -1214,6 +1239,27 @@ fn parseCommonArg(
return true;
}
if (std.mem.eql(u8, "--storage-engine", opt)) {
const str = args.next() orelse {
log.fatal(.app, "missing argument value", .{ .arg = "--storage-engine" });
return error.InvalidArgument;
};
common.storage_engine = std.meta.stringToEnum(Storage.EngineType, str) orelse {
log.fatal(.app, "invalid argument value", .{ .arg = opt, .val = str });
return error.InvalidArgument;
};
return true;
}
if (std.mem.eql(u8, "--storage-sqlite-path", opt)) {
const str = args.next() orelse {
log.fatal(.app, "missing argument value", .{ .arg = "--storage-sqlite-path" });
return error.InvalidArgument;
};
common.storage_sqlite_path = try allocator.dupeZ(u8, str);
return true;
}
if (std.mem.eql(u8, "--block-private-networks", opt)) {
common.block_private_networks = true;
return true;

1
src/log.zig
View File

@@ -41,6 +41,7 @@ pub const Scope = enum {
mcp,
cache,
websocket,
storage,
};
const Opts = struct {

13968
src/sqlite3.h Normal file
View File

File diff suppressed because it is too large Load Diff

63
src/storage/Storage.zig Normal file
View File

@@ -0,0 +1,63 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const log = @import("../log.zig");
const Config = @import("../Config.zig");
const Sqlite = @import("sqlite/Sqlite.zig");
const Allocator = std.mem.Allocator;
const Storage = @This();
pub const EngineType = enum {
sqlite,
};
const Engine = union(EngineType) {
sqlite: Sqlite,
};
engine: Engine,
pub fn init(allocator: Allocator, config: *const Config) !Storage {
const engine_type = config.storageEngine() orelse .sqlite;
const engine = initEngine(allocator, engine_type, config) catch |err| {
log.fatal(.storage, "storage setup", .{ .engine = engine_type, .err = err });
return err;
};
return .{
.engine = engine,
};
}
fn initEngine(allocator: Allocator, engine_type: EngineType, config: *const Config) !Engine {
switch (engine_type) {
.sqlite => {
const sqlite_path = config.storageSqlitePath();
return .{ .sqlite = try Sqlite.init(allocator, sqlite_path) };
},
}
}
pub fn deinit(self: *Storage, allocator: Allocator) void {
switch (self.engine) {
inline else => |*engine| engine.deinit(allocator),
}
}

167
src/storage/sqlite/Pool.zig Normal file
View File

@@ -0,0 +1,167 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const Sqlite = @import("Sqlite.zig");
const c = Sqlite.c;
const Thread = std.Thread;
const Allocator = std.mem.Allocator;
const Pool = @This();
available: usize,
mutex: Thread.Mutex,
cond: Thread.Condition,
conns: []Sqlite.Conn,
pub fn init(allocator: Allocator, path: [:0]const u8) !Pool {
// can't have a pool of connections to in-memory database, so, to keep the
// API simple, we create a pool of 1.
const count: usize = if (std.mem.eql(u8, path, ":memory:")) 1 else 5;
var conns = try allocator.alloc(Sqlite.Conn, count);
errdefer allocator.free(conns);
var initialized: usize = 0;
errdefer {
for (0..initialized) |i| {
conns[i].close();
}
}
for (0..count) |i| {
conns[i] = try Sqlite.Conn.open(path);
initialized += 1;
try conns[i].busyTimeout(1000);
}
return .{
.cond = .{},
.mutex = .{},
.conns = conns,
.available = count,
};
}
pub fn deinit(self: *Pool, allocator: Allocator) void {
for (self.conns) |conn| {
conn.close();
}
allocator.free(self.conns);
}
pub fn acquire(self: *Pool) !Sqlite.Conn {
const conns = self.conns;
self.mutex.lock();
while (true) {
const available = self.available;
if (available == 0) {
try self.cond.timedWait(&self.mutex, 5 * std.time.ns_per_s);
continue;
}
const index = available - 1;
const conn = conns[index];
self.available = index;
self.mutex.unlock();
return conn;
}
}
pub fn release(self: *Pool, conn: Sqlite.Conn) void {
var conns = self.conns;
self.mutex.lock();
const available = self.available;
conns[available] = conn;
self.available = available + 1;
self.mutex.unlock();
self.cond.signal();
}
const testing = @import("../../testing.zig");
test "Sqlite: Pool" {
// :memory: _has_ to run with a single connetion in the pool, which isn't
// that useful for testing. So we create a temp file.
std.fs.cwd().deleteFile("/tmp/lightpanda_test.sqlite") catch {};
var pool = try Pool.init(testing.allocator, "/tmp/lightpanda_test.sqlite");
defer {
pool.deinit(testing.allocator);
std.fs.cwd().deleteFile("/tmp/lightpanda_test.sqlite") catch {};
}
{
const conn = try pool.acquire();
defer pool.release(conn);
try conn.exec("create table pool_test (cnt int not null)", .{});
try conn.exec("insert into pool_test (cnt) values (0)", .{});
}
for (pool.conns) |conn| {
// This is not safe and can result in corruption. This is only set
// because the tests might be run on really slow hardware and we
// want to avoid having a busy timeout.
try conn.exec("pragma synchronous=off", .{});
// Also not safe, but we're trying to avoid busy timeouts without using
// WAL mode, which can trigger false positives in thread-sanitizer
try conn.exec("pragma journal_mode=memory", .{});
}
const t1 = try Thread.spawn(.{}, testPool, .{&pool});
const t2 = try Thread.spawn(.{}, testPool, .{&pool});
const t3 = try Thread.spawn(.{}, testPool, .{&pool});
const t4 = try Thread.spawn(.{}, testPool, .{&pool});
const t5 = try Thread.spawn(.{}, testPool, .{&pool});
const t6 = try Thread.spawn(.{}, testPool, .{&pool});
t1.join();
t2.join();
t3.join();
t4.join();
t5.join();
t6.join();
const c1 = try pool.acquire();
defer pool.release(c1);
const row = (try c1.row("select cnt from pool_test", .{})).?;
try testing.expectEqual(600, row.get(i64, 0));
row.deinit();
try c1.exec("drop table pool_test", .{});
}
fn testPool(p: *Pool) !void {
for (0..100) |_| {
const conn = try p.acquire();
conn.exec("begin immediate", .{}) catch unreachable;
conn.exec("update pool_test set cnt = cnt + 1", .{}) catch |err| {
std.debug.print("update err: {any}\n", .{err});
unreachable;
};
conn.exec("commit", .{}) catch unreachable;
p.release(conn);
std.Thread.sleep(2 * std.time.ns_per_ms);
}
}

576
src/storage/sqlite/Sqlite.zig Normal file
View File

@@ -0,0 +1,576 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const Pool = @import("Pool.zig");
pub const c = @cImport(@cInclude("sqlite3.h"));
const log = lp.log;
const Allocator = std.mem.Allocator;
const Sqlite = @This();
pool: Pool,
pub fn init(allocator: Allocator, path: ?[:0]const u8) !Sqlite {
var pool = try Pool.init(allocator, path orelse ":memory:");
errdefer pool.deinit(allocator);
{
// copy by value warning! The connection HAS to be returned to the
// pool in this scope. If we didn't have this scope, we'd assign the
// pool to the return value (copy A) and then release the original
const conn = try pool.acquire();
defer pool.release(conn);
try @import("migrations.zig").run(conn);
}
return .{
.pool = pool,
};
}
pub fn deinit(self: *Sqlite, allocator: Allocator) void {
self.pool.deinit(allocator);
}
pub const Conn = struct {
conn: *c.sqlite3,
pub fn open(path: [:0]const u8) !Conn {
var conn: ?*c.sqlite3 = null;
const flags = c.SQLITE_OPEN_READWRITE | c.SQLITE_OPEN_CREATE;
const rc = c.sqlite3_open_v2(path.ptr, &conn, flags, null);
if (rc != c.SQLITE_OK) {
if (conn) |connection| {
_ = c.sqlite3_close_v2(connection);
}
return errorFromCode(rc);
}
return .{ .conn = conn.? };
}
pub fn close(self: Conn) void {
_ = c.sqlite3_close_v2(self.conn);
}
pub fn exec(self: Conn, sql: [:0]const u8, values: anytype) !void {
if (values.len == 0) {
const rc = c.sqlite3_exec(self.conn, sql, null, null, null);
if (rc != c.SQLITE_OK) {
return errorFromCode(rc);
}
return;
}
const stmt = try self.prepare(sql);
defer stmt.deinit();
try stmt.bind(values);
try stmt.stepToCompletion();
}
pub fn scalar(self: Conn, comptime T: type, sql: []const u8, values: anytype) !?T {
if (comptime isScalar(T) == false) {
@compileError("Cannot use `sqlite.scalar` function with a non-scalar type. Who owns that memory?");
}
const stmt = try self.prepare(sql);
errdefer stmt.deinit();
try stmt.bind(values);
if (try stmt.step() == false) {
stmt.deinit();
return null;
}
var r = Row{ .stmt = stmt };
defer r.deinit();
return r.get(T, 0);
}
pub fn row(self: Conn, sql: []const u8, values: anytype) !?Row {
const stmt = try self.prepare(sql);
errdefer stmt.deinit();
try stmt.bind(values);
if (try stmt.step() == false) {
stmt.deinit();
return null;
}
return .{ .stmt = stmt };
}
pub fn rows(self: Conn, sql: []const u8, values: anytype) !Rows {
const stmt = try self.prepare(sql);
errdefer stmt.deinit();
try stmt.bind(values);
return .{ .stmt = stmt };
}
fn prepare(self: Conn, sql: []const u8) !Statement {
var stmt: ?*c.sqlite3_stmt = null;
var pz_tail: [*:0]const u8 = undefined;
const rc = c.sqlite3_prepare_v2(self.conn, sql.ptr, @intCast(sql.len), &stmt, @ptrCast(&pz_tail));
if (rc != c.SQLITE_OK) {
return errorFromCode(rc);
}
return .{ .stmt = stmt.?, .conn = self.conn };
}
pub fn busyTimeout(self: Conn, ms: c_int) !void {
const rc = c.sqlite3_busy_timeout(self.conn, ms);
if (rc != c.SQLITE_OK) {
return errorFromCode(rc);
}
}
pub fn lastError(self: Conn) [:0]const u8 {
return std.mem.span(c.sqlite3_errmsg(self.conn));
}
};
const Statement = struct {
conn: *c.sqlite3,
stmt: *c.sqlite3_stmt,
pub fn deinit(self: Statement) void {
_ = c.sqlite3_finalize(self.stmt);
}
pub fn bind(self: Statement, values: anytype) !void {
const stmt = self.stmt;
inline for (values, 0..) |value, i| {
try _bind(@TypeOf(value), stmt, value, i + 1);
}
}
pub fn get(self: Statement, comptime T: type, index: usize) T {
const stmt = self.stmt;
const TT = switch (@typeInfo(T)) {
.optional => |opt| blk: {
if (c.sqlite3_column_type(stmt, @intCast(index)) == c.SQLITE_NULL) {
return null;
}
break :blk opt.child;
},
else => T,
};
return switch (TT) {
i64 => @intCast(c.sqlite3_column_int64(stmt, @intCast(index))),
bool => @as(i64, @intCast(c.sqlite3_column_int64(stmt, @intCast(index)))) == 1,
f64 => @floatCast(c.sqlite3_column_double(stmt, @intCast(index))),
[]const u8 => {
const len = c.sqlite3_column_bytes(stmt, @intCast(index));
if (len == 0) {
return "";
}
const data = c.sqlite3_column_text(stmt, @intCast(index));
return @as([*c]const u8, @ptrCast(data))[0..@intCast(len)];
},
[:0]const u8 => {
const len = c.sqlite3_column_bytes(stmt, @intCast(index));
if (len == 0) {
return "";
}
const data = c.sqlite3_column_text(stmt, @intCast(index));
return @as([*c]const u8, @ptrCast(data))[0..@intCast(len) :0];
},
else => @compileError("unsupport column type: " ++ @typeName(T)),
};
}
pub fn step(self: Statement) !bool {
const s = self.stmt;
const rc = c.sqlite3_step(s);
if (rc == c.SQLITE_DONE) {
return false;
}
if (rc != c.SQLITE_ROW) {
return errorFromCode(rc);
}
return true;
}
pub fn stepToCompletion(self: Statement) !void {
const stmt = self.stmt;
while (true) {
switch (c.sqlite3_step(stmt)) {
c.SQLITE_DONE => return,
c.SQLITE_ROW => continue,
else => |rc| return errorFromCode(rc),
}
}
}
pub fn reset(self: Statement) !void {
switch (c.sqlite3_reset(self.stmt)) {
c.SQLITE_OK => return,
else => |rc| return errorFromCode(rc),
}
}
fn _bind(comptime T: type, stmt: *c.sqlite3_stmt, value: anytype, bind_index: c_int) !void {
var rc: c_int = 0;
switch (@typeInfo(T)) {
.null => rc = c.sqlite3_bind_null(stmt, bind_index),
.int, .comptime_int => rc = c.sqlite3_bind_int64(stmt, bind_index, @intCast(value)),
.float, .comptime_float => rc = c.sqlite3_bind_double(stmt, bind_index, value),
.bool => {
if (value) {
rc = c.sqlite3_bind_int64(stmt, bind_index, @intCast(1));
} else {
rc = c.sqlite3_bind_int64(stmt, bind_index, @intCast(0));
}
},
.pointer => |ptr| {
switch (ptr.size) {
.one => switch (@typeInfo(ptr.child)) {
.array => |arr| {
if (arr.child == u8) {
rc = c.sqlite3_bind_text(stmt, bind_index, value.ptr, @intCast(value.len), c.SQLITE_STATIC);
} else {
bindError(T);
}
},
else => bindError(T),
},
.slice => switch (ptr.child) {
u8 => rc = c.sqlite3_bind_text(stmt, bind_index, value.ptr, @intCast(value.len), c.SQLITE_STATIC),
else => bindError(T),
},
else => bindError(T),
}
},
.array => |arr| {
if (arr.child == u8) {
@compileError("Pass a string slice, rather than an array, to bind a text/blob. String arrays will be supported when https://github.com/ziglang/zig/issues/15893#issuecomment-1925092582 is fixed");
// const data: []const u8 = value[0..arr.len];
// rc = c.sqlite3_bind_text(stmt, bind_index, data.ptr, @intCast(data.len), c.SQLITE_TRANSIENT);
} else {
bindError(T);
}
},
.optional => |opt| {
if (value) |v| {
return _bind(opt.child, stmt, v, bind_index);
} else {
rc = c.sqlite3_bind_null(stmt, bind_index);
}
},
else => bindError(T),
}
if (rc != c.SQLITE_OK) {
return errorFromCode(rc);
}
}
fn bindError(comptime T: type) void {
@compileError("cannot bind value of type " ++ @typeName(T));
}
};
const Row = struct {
stmt: Statement,
pub fn deinit(self: Row) void {
self.stmt.deinit();
}
pub fn get(self: Row, comptime T: type, index: usize) T {
return self.stmt.get(T, index);
}
};
const Rows = struct {
stmt: Statement,
pub fn deinit(self: Rows) void {
self.stmt.deinit();
}
pub fn next(self: *Rows) !?Row {
const stmt = self.stmt;
const has_data = try stmt.step();
if (!has_data) {
return null;
}
return .{ .stmt = stmt };
}
};
pub fn errorFromCode(result: c_int) Error {
return switch (result) {
c.SQLITE_ABORT => error.Abort,
c.SQLITE_AUTH => error.Auth,
c.SQLITE_BUSY => error.Busy,
c.SQLITE_CANTOPEN => error.CantOpen,
c.SQLITE_CONSTRAINT => error.Constraint,
c.SQLITE_CORRUPT => error.Corrupt,
c.SQLITE_EMPTY => error.Empty,
c.SQLITE_ERROR => error.Error,
c.SQLITE_FORMAT => error.Format,
c.SQLITE_FULL => error.Full,
c.SQLITE_INTERNAL => error.Internal,
c.SQLITE_INTERRUPT => error.Interrupt,
c.SQLITE_IOERR => error.IoErr,
c.SQLITE_LOCKED => error.Locked,
c.SQLITE_MISMATCH => error.Mismatch,
c.SQLITE_MISUSE => error.Misuse,
c.SQLITE_NOLFS => error.NoLFS,
c.SQLITE_NOMEM => error.NoMem,
c.SQLITE_NOTADB => error.NotADB,
c.SQLITE_NOTFOUND => error.Notfound,
c.SQLITE_NOTICE => error.Notice,
c.SQLITE_PERM => error.Perm,
c.SQLITE_PROTOCOL => error.Protocol,
c.SQLITE_RANGE => error.Range,
c.SQLITE_READONLY => error.ReadOnly,
c.SQLITE_SCHEMA => error.Schema,
c.SQLITE_TOOBIG => error.TooBig,
c.SQLITE_WARNING => error.Warning,
// Extended codes
c.SQLITE_ERROR_MISSING_COLLSEQ => error.ErrorMissingCollseq,
c.SQLITE_ERROR_RETRY => error.ErrorRetry,
c.SQLITE_ERROR_SNAPSHOT => error.ErrorSnapshot,
c.SQLITE_IOERR_READ => error.IoerrRead,
c.SQLITE_IOERR_SHORT_READ => error.IoerrShortRead,
c.SQLITE_IOERR_WRITE => error.IoerrWrite,
c.SQLITE_IOERR_FSYNC => error.IoerrFsync,
c.SQLITE_IOERR_DIR_FSYNC => error.IoerrDir_fsync,
c.SQLITE_IOERR_TRUNCATE => error.IoerrTruncate,
c.SQLITE_IOERR_FSTAT => error.IoerrFstat,
c.SQLITE_IOERR_UNLOCK => error.IoerrUnlock,
c.SQLITE_IOERR_RDLOCK => error.IoerrRdlock,
c.SQLITE_IOERR_DELETE => error.IoerrDelete,
c.SQLITE_IOERR_BLOCKED => error.IoerrBlocked,
c.SQLITE_IOERR_NOMEM => error.IoerrNomem,
c.SQLITE_IOERR_ACCESS => error.IoerrAccess,
c.SQLITE_IOERR_CHECKRESERVEDLOCK => error.IoerrCheckreservedlock,
c.SQLITE_IOERR_LOCK => error.IoerrLock,
c.SQLITE_IOERR_CLOSE => error.IoerrClose,
c.SQLITE_IOERR_DIR_CLOSE => error.IoerrDirClose,
c.SQLITE_IOERR_SHMOPEN => error.IoerrShmopen,
c.SQLITE_IOERR_SHMSIZE => error.IoerrShmsize,
c.SQLITE_IOERR_SHMLOCK => error.IoerrShmlock,
c.SQLITE_IOERR_SHMMAP => error.IoerrShmmap,
c.SQLITE_IOERR_SEEK => error.IoerrSeek,
c.SQLITE_IOERR_DELETE_NOENT => error.IoerrDeleteNoent,
c.SQLITE_IOERR_MMAP => error.IoerrMmap,
c.SQLITE_IOERR_GETTEMPPATH => error.IoerrGetTempPath,
c.SQLITE_IOERR_CONVPATH => error.IoerrConvPath,
c.SQLITE_IOERR_VNODE => error.IoerrVnode,
c.SQLITE_IOERR_AUTH => error.IoerrAuth,
c.SQLITE_IOERR_BEGIN_ATOMIC => error.IoerrBeginAtomic,
c.SQLITE_IOERR_COMMIT_ATOMIC => error.IoerrCommitAtomic,
c.SQLITE_IOERR_ROLLBACK_ATOMIC => error.IoerrRollbackAtomic,
c.SQLITE_IOERR_DATA => error.IoerrData,
c.SQLITE_IOERR_CORRUPTFS => error.IoerrCorruptFS,
c.SQLITE_LOCKED_SHAREDCACHE => error.LockedSharedCache,
c.SQLITE_LOCKED_VTAB => error.LockedVTab,
c.SQLITE_BUSY_RECOVERY => error.BusyRecovery,
c.SQLITE_BUSY_SNAPSHOT => error.BusySnapshot,
c.SQLITE_BUSY_TIMEOUT => error.BusyTimeout,
c.SQLITE_CANTOPEN_NOTEMPDIR => error.CantOpenNoTempDir,
c.SQLITE_CANTOPEN_ISDIR => error.CantOpenIsDir,
c.SQLITE_CANTOPEN_FULLPATH => error.CantOpenFullPath,
c.SQLITE_CANTOPEN_CONVPATH => error.CantOpenConvPath,
c.SQLITE_CANTOPEN_DIRTYWAL => error.CantOpenDirtyWal,
c.SQLITE_CANTOPEN_SYMLINK => error.CantOpenSymlink,
c.SQLITE_CORRUPT_VTAB => error.CorruptVTab,
c.SQLITE_CORRUPT_SEQUENCE => error.CorruptSequence,
c.SQLITE_CORRUPT_INDEX => error.CorruptIndex,
c.SQLITE_READONLY_RECOVERY => error.ReadonlyRecovery,
c.SQLITE_READONLY_CANTLOCK => error.ReadonlyCantlock,
c.SQLITE_READONLY_ROLLBACK => error.ReadonlyRollback,
c.SQLITE_READONLY_DBMOVED => error.ReadonlyDbMoved,
c.SQLITE_READONLY_CANTINIT => error.ReadonlyCantInit,
c.SQLITE_READONLY_DIRECTORY => error.ReadonlyDirectory,
c.SQLITE_ABORT_ROLLBACK => error.AbortRollback,
c.SQLITE_CONSTRAINT_CHECK => error.ConstraintCheck,
c.SQLITE_CONSTRAINT_COMMITHOOK => error.ConstraintCommithook,
c.SQLITE_CONSTRAINT_FOREIGNKEY => error.ConstraintForeignKey,
c.SQLITE_CONSTRAINT_FUNCTION => error.ConstraintFunction,
c.SQLITE_CONSTRAINT_NOTNULL => error.ConstraintNotNull,
c.SQLITE_CONSTRAINT_PRIMARYKEY => error.ConstraintPrimaryKey,
c.SQLITE_CONSTRAINT_TRIGGER => error.ConstraintTrigger,
c.SQLITE_CONSTRAINT_UNIQUE => error.ConstraintUnique,
c.SQLITE_CONSTRAINT_VTAB => error.ConstraintVTab,
c.SQLITE_CONSTRAINT_ROWID => error.ConstraintRowId,
c.SQLITE_CONSTRAINT_PINNED => error.ConstraintPinned,
c.SQLITE_CONSTRAINT_DATATYPE => error.ConstraintDatatype,
c.SQLITE_NOTICE_RECOVER_WAL => error.NoticeRecoverWal,
c.SQLITE_NOTICE_RECOVER_ROLLBACK => error.NoticeRecoverRollback,
c.SQLITE_WARNING_AUTOINDEX => error.WarningAutoIndex,
c.SQLITE_AUTH_USER => error.AuthUser,
c.SQLITE_OK_LOAD_PERMANENTLY => error.OkLoadPermanently,
else => {
log.err(.storage, "unknown error", .{ .engine = "sqlite", .code = result });
return error.Unknown;
},
};
}
pub const Error = error{
Abort,
Auth,
Busy,
CantOpen,
Constraint,
Corrupt,
Empty,
Error,
Format,
Full,
Internal,
Interrupt,
IoErr,
Locked,
Mismatch,
Misuse,
NoLFS,
NoMem,
NotADB,
Notfound,
Notice,
Perm,
Protocol,
Range,
ReadOnly,
Schema,
TooBig,
Warning,
ErrorMissingCollseq,
ErrorRetry,
ErrorSnapshot,
IoerrRead,
IoerrShortRead,
IoerrWrite,
IoerrFsync,
IoerrDir_fsync,
IoerrTruncate,
IoerrFstat,
IoerrUnlock,
IoerrRdlock,
IoerrDelete,
IoerrBlocked,
IoerrNomem,
IoerrAccess,
IoerrCheckreservedlock,
IoerrLock,
IoerrClose,
IoerrDirClose,
IoerrShmopen,
IoerrShmsize,
IoerrShmlock,
IoerrShmmap,
IoerrSeek,
IoerrDeleteNoent,
IoerrMmap,
IoerrGetTempPath,
IoerrConvPath,
IoerrVnode,
IoerrAuth,
IoerrBeginAtomic,
IoerrCommitAtomic,
IoerrRollbackAtomic,
IoerrData,
IoerrCorruptFS,
LockedSharedCache,
LockedVTab,
BusyRecovery,
BusySnapshot,
BusyTimeout,
CantOpenNoTempDir,
CantOpenIsDir,
CantOpenFullPath,
CantOpenConvPath,
CantOpenDirtyWal,
CantOpenSymlink,
CorruptVTab,
CorruptSequence,
CorruptIndex,
ReadonlyRecovery,
ReadonlyCantlock,
ReadonlyRollback,
ReadonlyDbMoved,
ReadonlyCantInit,
ReadonlyDirectory,
AbortRollback,
ConstraintCheck,
ConstraintCommithook,
ConstraintForeignKey,
ConstraintFunction,
ConstraintNotNull,
ConstraintPrimaryKey,
ConstraintTrigger,
ConstraintUnique,
ConstraintVTab,
ConstraintRowId,
ConstraintPinned,
ConstraintDatatype,
NoticeRecoverWal,
NoticeRecoverRollback,
WarningAutoIndex,
AuthUser,
OkLoadPermanently,
Unknown,
};
fn isScalar(comptime T: type) bool {
const TT = switch (@typeInfo(T)) {
.optional => |opt| opt.child,
else => T,
};
return TT == i64 or TT == bool or TT == f64;
}
const testing = @import("../../testing.zig");
test "Sqlite: exec, row and scalar" {
var conn = try Sqlite.Conn.open(":memory:");
defer conn.close();
try conn.exec("create table test (id integer primary key, name text, data blob)", .{});
try conn.exec("insert into test (name, data) values (?1, ?2)", .{ "test name", "binary data" });
{
var row = (try conn.row("select name, data from test where id = 1", .{})) orelse unreachable;
defer row.deinit();
try testing.expectEqual("test name", row.get([]const u8, 0));
}
{
try testing.expectEqual(1, (try conn.scalar(i64, "select count(*) from test where id = 1", .{})).?);
}
}
test "Sqlite: Migration" {
var sqlite = try Sqlite.init(testing.allocator, ":memory:");
defer sqlite.deinit(testing.allocator);
const conn = try sqlite.pool.acquire();
defer sqlite.pool.release(conn);
try testing.expectEqual(1, (try conn.scalar(i64, "select max(id) from migrations", .{})).?);
}

62
src/storage/sqlite/migrations.zig Normal file
View File

@@ -0,0 +1,62 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const lp = @import("lightpanda");
const Sqlite = @import("Sqlite.zig");
const log = lp.log;
pub fn run(conn: Sqlite.Conn) !void {
const version = try getVersion(conn);
log.info(.storage, "migration version", .{ .engine = "sqlite", .version = version });
}
fn getVersion(conn: Sqlite.Conn) !i64 {
const exists_sql = "select exists (select 1 from sqlite_schema where type='table' and name='migrations')";
if (try conn.scalar(bool, exists_sql, .{}) orelse false) {
if (try conn.scalar(i64, "select max(id) from migrations", .{})) |version| {
return version;
}
log.fatal(.storage, "corrupt database", .{ .engine = "sqlite", .note = "The sqlite database has an existing but empty `migrations` table" });
return error.CorruptDatabase;
}
// this pragma is one of the the few (if not only) one that's persisted, so
// we only have to do it the first time.
conn.exec("pragma journal_mode=wal", .{}) catch |err| {
log.fatal(.storage, "migrate", .{
.err = err,
.step = "journal_mode",
.sqlite = conn.lastError(),
});
return err;
};
const create_sql =
\\ create table migrations as
\\ select 1 as id, current_timestamp as created_at
;
conn.exec(create_sql, .{}) catch |err| {
log.fatal(.storage, "migrate", .{ .err = err, .sqlite = conn.lastError(), .step = "create migrations" });
return err;
};
return 1;
}