mirror of
https://github.com/navidrome/navidrome.git
synced 2026-03-09 10:17:46 -04:00
2471bb9cf6
* feat(plugins): add expires_at column to kvstore schema * feat(plugins): filter expired keys in kvstore Get, Has, List * feat(plugins): add periodic cleanup of expired kvstore keys * feat(plugins): add SetWithTTL, DeleteByPrefix, and GetMany to kvstore Add three new methods to the KVStore host service: - SetWithTTL: store key-value pairs with automatic expiration - DeleteByPrefix: remove all keys matching a prefix in one operation - GetMany: retrieve multiple values in a single call All methods include comprehensive unit tests covering edge cases, expiration behavior, size tracking, and LIKE-special characters. * feat(plugins): regenerate code and update test plugin for new kvstore methods Regenerate host function wrappers and PDK bindings for Go, Python, and Rust. Update the test-kvstore plugin to exercise SetWithTTL, DeleteByPrefix, and GetMany. * feat(plugins): add integration tests for new kvstore methods Add WASM integration tests for SetWithTTL, DeleteByPrefix, and GetMany operations through the plugin boundary, verifying end-to-end behavior including TTL expiration, prefix deletion, and batch retrieval. * fix(plugins): address lint issues in kvstore implementation Handle tx.Rollback error return and suppress gosec false positive for parameterized SQL query construction in GetMany. * fix(plugins): Set clears expires_at when overwriting a TTL'd key Previously, calling Set() on a key that was stored with SetWithTTL() would leave the expires_at value intact, causing the key to silently expire even though Set implies permanent storage. Also excludes expired keys from currentSize calculation at startup. * refactor(plugins): simplify kvstore by removing in-memory size cache Replaced the in-memory currentSize cache (atomic.Int64), periodic cleanup timer, and mutex with direct database queries for storage accounting. This eliminates race conditions and cache drift issues at negligible performance cost for plugin-sized datasets. Also unified Set and SetWithTTL into a shared setValue method, simplified DeleteByPrefix to use RowsAffected instead of a transaction, and added an index on expires_at for efficient expiration filtering. * feat(plugins): add generic SQLite migration helper and refactor kvstore schema Add a reusable migrateDB helper that tracks schema versions via SQLite's PRAGMA user_version and applies pending migrations transactionally. Replace the ad-hoc createKVStoreSchema function in kvstore with a declarative migrations slice, making it easy to add future schema changes. Remove the now-redundant schema migration test since migrateDB has its own test suite and every kvstore test exercises the migrations implicitly. Signed-off-by: Deluan <deluan@navidrome.org> * fix(plugins): harden kvstore with explicit NULL handling, prefix validation, and cleanup timeout - Use sql.NullString for expires_at to explicitly send NULL instead of relying on datetime('now', '') returning NULL by accident - Reject empty prefix in DeleteByPrefix to prevent accidental data wipe - Add 5s timeout context to cleanupExpired on Close - Replace time.Sleep in unit tests with pre-expired timestamps Signed-off-by: Deluan <deluan@navidrome.org> * refactor(plugins): use batch processing in GetMany Process keys in chunks of 200 using slice.CollectChunks to avoid hitting SQLite's SQLITE_MAX_VARIABLE_NUMBER limit with large key sets. * feat(plugins): add periodic cleanup goroutine for expired kvstore keys Use the manager's context to control a background goroutine that purges expired keys every hour, stopping naturally on shutdown when the context is cancelled. --------- Signed-off-by: Deluan <deluan@navidrome.org>
363 lines
9.8 KiB
Python
363 lines
9.8 KiB
Python
# Code generated by ndpgen. DO NOT EDIT.
|
|
#
|
|
# This file contains client wrappers for the KVStore host service.
|
|
# It is intended for use in Navidrome plugins built with extism-py.
|
|
#
|
|
# IMPORTANT: Due to a limitation in extism-py, you cannot import this file directly.
|
|
# The @extism.import_fn decorators are only detected when defined in the plugin's
|
|
# main __init__.py file. Copy the needed functions from this file into your plugin.
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import extism
|
|
import json
|
|
import base64
|
|
|
|
|
|
class HostFunctionError(Exception):
|
|
"""Raised when a host function returns an error."""
|
|
pass
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_set")
|
|
def _kvstore_set(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_setwithttl")
|
|
def _kvstore_setwithttl(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_get")
|
|
def _kvstore_get(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_getmany")
|
|
def _kvstore_getmany(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_has")
|
|
def _kvstore_has(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_list")
|
|
def _kvstore_list(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_delete")
|
|
def _kvstore_delete(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_deletebyprefix")
|
|
def _kvstore_deletebyprefix(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@extism.import_fn("extism:host/user", "kvstore_getstorageused")
|
|
def _kvstore_getstorageused(offset: int) -> int:
|
|
"""Raw host function - do not call directly."""
|
|
...
|
|
|
|
|
|
@dataclass
|
|
class KVStoreGetResult:
|
|
"""Result type for kvstore_get."""
|
|
value: bytes
|
|
exists: bool
|
|
|
|
|
|
def kvstore_set(key: str, value: bytes) -> None:
|
|
"""Set stores a byte value with the given key.
|
|
|
|
Parameters:
|
|
- key: The storage key (max 256 bytes, UTF-8)
|
|
- value: The byte slice to store
|
|
|
|
Returns an error if the storage limit would be exceeded or the operation fails.
|
|
|
|
Args:
|
|
key: str parameter.
|
|
value: bytes parameter.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"key": key,
|
|
"value": base64.b64encode(value).decode("ascii"),
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_set(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
|
|
|
|
def kvstore_set_with_ttl(key: str, value: bytes, ttl_seconds: int) -> None:
|
|
"""SetWithTTL stores a byte value with the given key and a time-to-live.
|
|
|
|
After ttlSeconds, the key is treated as non-existent and will be
|
|
cleaned up lazily. ttlSeconds must be greater than 0.
|
|
|
|
Parameters:
|
|
- key: The storage key (max 256 bytes, UTF-8)
|
|
- value: The byte slice to store
|
|
- ttlSeconds: Time-to-live in seconds (must be > 0)
|
|
|
|
Returns an error if the storage limit would be exceeded or the operation fails.
|
|
|
|
Args:
|
|
key: str parameter.
|
|
value: bytes parameter.
|
|
ttl_seconds: int parameter.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"key": key,
|
|
"value": base64.b64encode(value).decode("ascii"),
|
|
"ttlSeconds": ttl_seconds,
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_setwithttl(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
|
|
|
|
def kvstore_get(key: str) -> KVStoreGetResult:
|
|
"""Get retrieves a byte value from storage.
|
|
|
|
Parameters:
|
|
- key: The storage key
|
|
|
|
Returns the value and whether the key exists.
|
|
|
|
Args:
|
|
key: str parameter.
|
|
|
|
Returns:
|
|
KVStoreGetResult containing value, exists,.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"key": key,
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_get(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
return KVStoreGetResult(
|
|
value=base64.b64decode(response.get("value", "")),
|
|
exists=response.get("exists", False),
|
|
)
|
|
|
|
|
|
def kvstore_get_many(keys: Any) -> Any:
|
|
"""GetMany retrieves multiple values in a single call.
|
|
|
|
Parameters:
|
|
- keys: The storage keys to retrieve
|
|
|
|
Returns a map of key to value for keys that exist and have not expired.
|
|
Missing or expired keys are omitted from the result.
|
|
|
|
Args:
|
|
keys: Any parameter.
|
|
|
|
Returns:
|
|
Any: The result value.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"keys": keys,
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_getmany(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
return response.get("values", None)
|
|
|
|
|
|
def kvstore_has(key: str) -> bool:
|
|
"""Has checks if a key exists in storage.
|
|
|
|
Parameters:
|
|
- key: The storage key
|
|
|
|
Returns true if the key exists.
|
|
|
|
Args:
|
|
key: str parameter.
|
|
|
|
Returns:
|
|
bool: The result value.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"key": key,
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_has(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
return response.get("exists", False)
|
|
|
|
|
|
def kvstore_list(prefix: str) -> Any:
|
|
"""List returns all keys matching the given prefix.
|
|
|
|
Parameters:
|
|
- prefix: Key prefix to filter by (empty string returns all keys)
|
|
|
|
Returns a slice of matching keys.
|
|
|
|
Args:
|
|
prefix: str parameter.
|
|
|
|
Returns:
|
|
Any: The result value.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"prefix": prefix,
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_list(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
return response.get("keys", None)
|
|
|
|
|
|
def kvstore_delete(key: str) -> None:
|
|
"""Delete removes a value from storage.
|
|
|
|
Parameters:
|
|
- key: The storage key
|
|
|
|
Returns an error if the operation fails. Does not return an error if the key doesn't exist.
|
|
|
|
Args:
|
|
key: str parameter.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"key": key,
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_delete(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
|
|
|
|
def kvstore_delete_by_prefix(prefix: str) -> int:
|
|
"""DeleteByPrefix removes all keys matching the given prefix.
|
|
|
|
Parameters:
|
|
- prefix: Key prefix to match (must not be empty)
|
|
|
|
Returns the number of keys deleted. Includes expired keys.
|
|
|
|
Args:
|
|
prefix: str parameter.
|
|
|
|
Returns:
|
|
int: The result value.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request = {
|
|
"prefix": prefix,
|
|
}
|
|
request_bytes = json.dumps(request).encode("utf-8")
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_deletebyprefix(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
return response.get("deletedCount", 0)
|
|
|
|
|
|
def kvstore_get_storage_used() -> int:
|
|
"""GetStorageUsed returns the total storage used by this plugin in bytes.
|
|
|
|
Returns:
|
|
int: The result value.
|
|
|
|
Raises:
|
|
HostFunctionError: If the host function returns an error.
|
|
"""
|
|
request_bytes = b"{}"
|
|
request_mem = extism.memory.alloc(request_bytes)
|
|
response_offset = _kvstore_getstorageused(request_mem.offset)
|
|
response_mem = extism.memory.find(response_offset)
|
|
response = json.loads(extism.memory.string(response_mem))
|
|
|
|
if response.get("error"):
|
|
raise HostFunctionError(response["error"])
|
|
|
|
return response.get("bytes", 0)
|