Enhance ephemeral indexing and add filesystem watching support

- Updated `EphemeralIndex` to preserve explicitly browsed subdirectories during re-indexing, preventing loss of user navigation context.
- Modified `clear_directory_children` to return the count of cleared entries and a list of deleted browsed directories.
- Introduced `EphemeralIndexCache` enhancements to support filesystem watching, allowing paths to be monitored for changes.
- Added methods for registering, unregistering, and checking watched paths, improving the responsiveness of the indexing system.
- Updated documentation and tests to reflect new functionality and ensure reliability.
This commit is contained in:
Jamie Pine
2025-12-08 00:38:04 -08:00
parent b6779d71ac
commit 3739b3f34f
5 changed files with 354 additions and 196 deletions

View File

@@ -100,11 +100,23 @@ impl EphemeralIndexCache {
/// Clear stale entries for a path before re-indexing (async version)
///
/// Call this after create_for_indexing to remove old children entries.
/// This prevents ghost entries when files are deleted between index runs.
/// Removes files and unbrowsed subdirectories, preserving subdirectories
/// that were explicitly navigated to. Verifies preserved directories still
/// exist on the filesystem and removes deleted ones from tracking.
pub async fn clear_for_reindex(&self, path: &Path) -> usize {
let indexed = self.indexed_paths.read().clone();
let mut index = self.index.write().await;
index.clear_directory_children(path)
let (cleared, deleted_browsed_dirs) = index.clear_directory_children(path, &indexed);
// Remove deleted browsed directories from indexed_paths
if !deleted_browsed_dirs.is_empty() {
let mut indexed_paths = self.indexed_paths.write();
for deleted_path in deleted_browsed_dirs {
indexed_paths.remove(&deleted_path);
}
}
cleared
}
/// Mark indexing as complete for a path

View File

@@ -451,44 +451,86 @@ impl EphemeralIndex {
)
}
/// Clears immediate children of a directory to prepare for re-indexing.
/// Clears entries before re-indexing, preserving explicitly browsed subdirectories.
///
/// This prevents ghost entries when files are deleted between index runs.
/// The arena nodes become orphaned but remain allocated, which is acceptable
/// for ephemeral indexes since memory pressure triggers full eviction anyway.
/// Only clears the direct children (non-recursive).
pub fn clear_directory_children(&mut self, dir_path: &Path) -> usize {
let children_paths: Vec<PathBuf> = if let Some(dir_id) = self.path_index.get(dir_path) {
if let Some(dir_node) = self.arena.get(*dir_id) {
dir_node
.children
.iter()
.filter_map(|&child_id| self.reconstruct_path(child_id))
.collect()
} else {
return 0;
}
} else {
return 0;
/// Since ephemeral indexing is shallow, subdirectories that were explicitly
/// navigated to (in `indexed_paths`) should be preserved as separate index
/// branches. Unbrowsed subdirectories are refreshed with the parent.
///
/// Returns (cleared_count, deleted_browsed_dirs) where deleted_browsed_dirs
/// contains paths that were in indexed_paths but no longer exist on disk.
pub fn clear_directory_children(
&mut self,
dir_path: &Path,
indexed_paths: &std::collections::HashSet<std::path::PathBuf>,
) -> (usize, Vec<std::path::PathBuf>) {
let dir_id = match self.path_index.get(dir_path) {
Some(&id) => id,
None => return (0, Vec::new()),
};
let mut cleared = 0;
let dir_node = match self.arena.get(dir_id) {
Some(node) => node,
None => return (0, Vec::new()),
};
for child_path in &children_paths {
if self.path_index.remove(child_path).is_some() {
cleared += 1;
}
let mut deleted_browsed_dirs = Vec::new();
// Collect children to remove
let mut children_to_remove: Vec<(PathBuf, super::ephemeral::EntryId)> = dir_node
.children
.iter()
.filter_map(|&child_id| {
let child_node = self.arena.get(child_id)?;
let child_path = self.reconstruct_path(child_id)?;
// Preserve subdirectories that were explicitly browsed AND still exist
if child_node.is_directory() && indexed_paths.contains(&child_path) {
// Verify the directory still exists on the filesystem
if std::fs::metadata(&child_path).is_ok() {
return None; // Preserve - still exists and was browsed
}
// Directory was deleted - track for removal from indexed_paths
tracing::debug!(
"Removing deleted browsed directory: {}",
child_path.display()
);
deleted_browsed_dirs.push(child_path.clone());
}
// Remove everything else (files, unbrowsed directories, deleted directories)
Some((child_path, child_id))
})
.collect();
let cleared = children_to_remove.len();
// Remove from indexes
for (child_path, _) in &children_to_remove {
self.path_index.remove(child_path);
self.entry_uuids.remove(child_path);
self.content_kinds.remove(child_path);
}
if let Some(dir_id) = self.path_index.get(dir_path) {
if let Some(dir_node) = self.arena.get_mut(*dir_id) {
dir_node.children.clear();
}
// Update parent's children list
if let Some(dir_node) = self.arena.get_mut(dir_id) {
let removed_ids: std::collections::HashSet<_> =
children_to_remove.iter().map(|(_, id)| id).collect();
dir_node
.children
.retain(|child_id| !removed_ids.contains(child_id));
}
cleared
if cleared > 0 {
tracing::debug!(
"Cleared {} entries from {} (preserved browsed subdirs)",
cleared,
dir_path.display()
);
}
(cleared, deleted_browsed_dirs)
}
fn reconstruct_path(&self, id: super::ephemeral::EntryId) -> Option<PathBuf> {

View File

@@ -349,22 +349,34 @@ const jobOptions: JobOption[] = [
export function useAddStorageDialog(
onStorageAdded?: (id: string) => void,
initialPath?: string,
) {
return dialogManager.create((props) => (
<AddStorageDialog {...props} onStorageAdded={onStorageAdded} />
<AddStorageDialog
{...props}
onStorageAdded={onStorageAdded}
initialPath={initialPath}
/>
));
}
function AddStorageDialog(props: {
id: number;
onStorageAdded?: (id: string) => void;
initialPath?: string;
}) {
const dialog = useDialog(props);
const platform = usePlatform();
const [step, setStep] = useState<ModalStep>("category");
// Derive initial folder name from path
const initialFolderName =
props.initialPath?.split("/").filter(Boolean).pop() || "";
const [step, setStep] = useState<ModalStep>(
props.initialPath ? "local-config" : "category",
);
const [selectedCategory, setSelectedCategory] =
useState<StorageCategory | null>(null);
useState<StorageCategory | null>(props.initialPath ? "local" : null);
const [selectedProvider, setSelectedProvider] =
useState<CloudProvider | null>(null);
const [tab, setTab] = useState<SettingsTab>("preset");
@@ -385,8 +397,8 @@ function AddStorageDialog(props: {
const localForm = useForm<LocalFolderFormData>({
defaultValues: {
path: "",
name: "",
path: props.initialPath || "",
name: initialFolderName,
mode: "Deep",
},
});
@@ -404,7 +416,9 @@ function AddStorageDialog(props: {
const currentMode = localForm.watch("mode");
const [selectedJobs, setSelectedJobs] = useState<Set<string>>(
new Set(
jobOptions.filter((j) => j.presets.includes("Deep")).map((j) => j.id),
jobOptions
.filter((j) => j.presets.includes("Deep"))
.map((j) => j.id),
),
);
@@ -539,7 +553,9 @@ function AddStorageDialog(props: {
localForm.setError("root", {
type: "manual",
message:
error instanceof Error ? error.message : "Failed to add location",
error instanceof Error
? error.message
: "Failed to add location",
});
}
});
@@ -692,7 +708,11 @@ function AddStorageDialog(props: {
"border-app-line bg-app-box hover:bg-app-hover hover:border-accent/50",
)}
>
<img src={category.icon} className="size-12" alt="" />
<img
src={category.icon}
className="size-12"
alt=""
/>
<div className="text-center">
<div className="text-sm font-medium text-ink">
{category.label}
@@ -733,7 +753,11 @@ function AddStorageDialog(props: {
"border-app-line bg-app-box hover:bg-app-hover hover:border-accent/50",
)}
>
<img src={provider.icon} className="size-10" alt="" />
<img
src={provider.icon}
className="size-10"
alt=""
/>
<div className="text-xs font-medium text-ink text-center">
{provider.name}
</div>
@@ -761,8 +785,9 @@ function AddStorageDialog(props: {
<div className="rounded-lg bg-accent/10 border border-accent/20 p-4 text-sm text-ink">
<strong>Coming Soon</strong>
<p className="mt-1 text-ink-dull">
Network protocol support (SMB, NFS, SFTP, WebDAV) is currently in
development. Check back in a future update!
Network protocol support (SMB, NFS, SFTP, WebDAV) is
currently in development. Check back in a future
update!
</p>
</div>
<div className="grid grid-cols-2 gap-3 opacity-50 pointer-events-none">
@@ -776,7 +801,11 @@ function AddStorageDialog(props: {
"border-app-line bg-app-box",
)}
>
<img src={protocol.icon} className="size-8" alt="" />
<img
src={protocol.icon}
className="size-8"
alt=""
/>
<div className="text-left">
<div className="text-sm font-medium text-ink">
{protocol.name}
@@ -820,17 +849,27 @@ function AddStorageDialog(props: {
"border-app-line bg-app-box hover:bg-app-hover hover:border-accent/50",
)}
>
<img src={HDDIcon} className="size-8" alt="" />
<img
src={HDDIcon}
className="size-8"
alt=""
/>
<div className="flex-1 min-w-0">
<div className="text-sm font-medium text-ink truncate">
{volume.name}
</div>
<div className="text-xs text-ink-faint">
{volume.mount_point} {volume.filesystem}
{volume.mount_point} {" "}
{volume.filesystem}
</div>
</div>
<div className="text-xs text-ink-dull">
{volume.total_capacity ? (volume.total_capacity / 1e9).toFixed(0) : '?'} GB
{volume.total_capacity
? (
volume.total_capacity / 1e9
).toFixed(0)
: "?"}{" "}
GB
</div>
</button>
))}
@@ -838,8 +877,8 @@ function AddStorageDialog(props: {
) : (
<div className="rounded-lg bg-app-box border border-app-line p-6 text-center">
<p className="text-sm text-ink-dull">
No untracked external drives found. Connect a drive and refresh
to see it here.
No untracked external drives found. Connect a
drive and refresh to see it here.
</p>
</div>
)}
@@ -867,7 +906,9 @@ function AddStorageDialog(props: {
<div className="relative">
<Input
value={localForm.watch("path") || ""}
onChange={(e) => localForm.setValue("path", e.target.value)}
onChange={(e) =>
localForm.setValue("path", e.target.value)
}
placeholder="Select a custom folder"
size="lg"
className="pr-14"
@@ -880,34 +921,40 @@ function AddStorageDialog(props: {
</div>
</div>
{suggestedLocations && suggestedLocations.locations.length > 0 && (
<div className="space-y-2">
<Label>Suggested Locations</Label>
<div className="grid grid-cols-2 gap-2 max-h-[280px] overflow-y-auto pr-1">
{suggestedLocations.locations.map((loc) => (
<button
key={loc.path}
type="button"
onClick={() => handleSelectSuggested(loc.path, loc.name)}
className="flex items-center gap-3 rounded-lg border border-app-line bg-app-box p-3 text-left transition-all hover:bg-app-hover hover:border-accent/50 h-fit"
>
<Folder
className="size-5 shrink-0 text-accent"
weight="fill"
/>
<div className="min-w-0 flex-1">
<div className="text-sm font-medium text-ink truncate">
{loc.name}
{suggestedLocations &&
suggestedLocations.locations.length > 0 && (
<div className="space-y-2">
<Label>Suggested Locations</Label>
<div className="grid grid-cols-2 gap-2 max-h-[280px] overflow-y-auto pr-1">
{suggestedLocations.locations.map((loc) => (
<button
key={loc.path}
type="button"
onClick={() =>
handleSelectSuggested(
loc.path,
loc.name,
)
}
className="flex items-center gap-3 rounded-lg border border-app-line bg-app-box p-3 text-left transition-all hover:bg-app-hover hover:border-accent/50 h-fit"
>
<Folder
className="size-5 shrink-0 text-accent"
weight="fill"
/>
<div className="min-w-0 flex-1">
<div className="text-sm font-medium text-ink truncate">
{loc.name}
</div>
<div className="text-xs text-ink-faint truncate">
{loc.path}
</div>
</div>
<div className="text-xs text-ink-faint truncate">
{loc.path}
</div>
</div>
</button>
))}
</button>
))}
</div>
</div>
</div>
)}
)}
</div>
</StorageDialog>
);
@@ -939,11 +986,16 @@ function AddStorageDialog(props: {
/>
</div>
<Tabs.Root value={tab} onValueChange={(v) => setTab(v as SettingsTab)}>
<Tabs.Root
value={tab}
onValueChange={(v) => setTab(v as SettingsTab)}
>
<Tabs.List>
<Tabs.Trigger value="preset">Preset</Tabs.Trigger>
<Tabs.Trigger value="jobs">
Jobs {selectedJobs.size > 0 && `(${selectedJobs.size})`}
Jobs{" "}
{selectedJobs.size > 0 &&
`(${selectedJobs.size})`}
</Tabs.Trigger>
</Tabs.List>
@@ -952,12 +1004,15 @@ function AddStorageDialog(props: {
<Label>Indexing Mode</Label>
<div className="grid grid-cols-3 gap-2">
{indexModes.map((mode) => {
const isSelected = currentMode === mode.value;
const isSelected =
currentMode === mode.value;
return (
<button
key={mode.value}
type="button"
onClick={() => handleModeChange(mode.value)}
onClick={() =>
handleModeChange(mode.value)
}
className={clsx(
"rounded-lg border p-3 text-left transition-all",
isSelected
@@ -981,17 +1036,21 @@ function AddStorageDialog(props: {
<Tabs.Content value="jobs" className="pt-3">
<div className="space-y-3 max-h-[280px] overflow-y-auto pr-1">
<p className="text-xs text-ink-faint">
Select which jobs to run after indexing. Extensions can add
more jobs.
Select which jobs to run after indexing.
Extensions can add more jobs.
</p>
<div className="grid grid-cols-2 gap-2">
{jobOptions.map((job) => {
const isSelected = selectedJobs.has(job.id);
const isSelected = selectedJobs.has(
job.id,
);
return (
<button
key={job.id}
type="button"
onClick={() => toggleJob(job.id)}
onClick={() =>
toggleJob(job.id)
}
className={clsx(
"flex items-start gap-2 rounded-lg border p-3 text-left transition-all",
isSelected
@@ -1110,8 +1169,10 @@ function AddStorageDialog(props: {
<div className="space-y-2">
<Label>
Endpoint
{provider.id === "r2" && " (e.g., https://account.r2.cloudflarestorage.com)"}
{provider.id === "minio" && " (e.g., http://localhost:9000)"}
{provider.id === "r2" &&
" (e.g., https://account.r2.cloudflarestorage.com)"}
{provider.id === "minio" &&
" (e.g., http://localhost:9000)"}
</Label>
<Input
{...cloudForm.register("endpoint")}

View File

@@ -7,8 +7,17 @@ import { getDeviceIconBySlug, useLibraryMutation } from "@sd/ts-client";
import { sdPathToUri } from "../utils";
import LaptopIcon from "@sd/assets/icons/Laptop.png";
import { useNormalizedQuery } from "@sd/ts-client";
import { TopBarButton, Popover, usePopover, PopoverContainer, PopoverSection, PopoverDivider } from "@sd/ui";
import {
TopBarButton,
Popover,
usePopover,
PopoverContainer,
PopoverSection,
PopoverDivider,
Button,
} from "@sd/ui";
import { useSelection } from "../SelectionContext";
import { useAddStorageDialog } from "./AddStorageModal";
interface PathBarProps {
path: SdPath;
@@ -118,77 +127,113 @@ function IndexIndicator({ path }: { path: SdPath }) {
// Find location with longest matching prefix
return locations
.filter((loc) => {
if (!loc.sd_path || !("Physical" in loc.sd_path)) return false;
if (!loc.sd_path || !("Physical" in loc.sd_path))
return false;
const locPath = loc.sd_path.Physical.path;
return pathStr.startsWith(locPath);
})
.sort((a, b) => {
const aPath = ("Physical" in a.sd_path!) ? a.sd_path!.Physical.path : "";
const bPath = ("Physical" in b.sd_path!) ? b.sd_path!.Physical.path : "";
const aPath =
"Physical" in a.sd_path!
? a.sd_path!.Physical.path
: "";
const bPath =
"Physical" in b.sd_path!
? b.sd_path!.Physical.path
: "";
return bPath.length - aPath.length;
})[0];
}
return undefined;
})();
if (!matchingLocation) return null;
const isIndexed = matchingLocation.index_mode !== "none";
const isIndexed =
matchingLocation?.index_mode !== undefined &&
matchingLocation.index_mode !== "none";
return (
<>
<Popover
popover={popover}
trigger={
<TopBarButton
icon={Eye}
active={isIndexed}
title={isIndexed ? "Location is indexed" : "Location not indexed"}
className={isIndexed ? "!text-blue-500" : undefined}
title={isIndexed ? "Location is indexed" : "Not indexed"}
/>
}
>
<PopoverContainer>
<PopoverSection>
<div className="px-2 py-1.5">
<div className="text-xs font-semibold text-ink">{matchingLocation.name ?? "Unknown"}</div>
<div className="text-xs text-ink-dull mt-0.5">
{isIndexed ? `Indexed (${matchingLocation.index_mode})` : "Not indexed"}
{matchingLocation ? (
<>
<PopoverSection>
<div className="px-2 py-1.5">
<div className="text-xs font-semibold text-ink">
{matchingLocation.name}
</div>
<div className="text-xs text-ink-dull mt-0.5">
{isIndexed
? `Indexed (${matchingLocation.index_mode})`
: "Not indexed"}
</div>
</div>
</PopoverSection>
<PopoverDivider />
<PopoverSection>
{!isIndexed && (
<button
onClick={async () => {
await enableIndexing.mutateAsync({
id: matchingLocation.id,
index_mode: "deep",
});
popover.setOpen(false);
}}
className="flex items-center gap-2 px-2 py-1.5 rounded-md text-xs font-medium text-ink hover:bg-app-hover transition-colors"
>
<Eye size={16} />
Enable Indexing
</button>
)}
<button
onClick={() => {
clearSelection();
popover.setOpen(false);
}}
className="flex items-center gap-2 px-2 py-1.5 rounded-md text-xs font-medium text-ink hover:bg-app-hover transition-colors"
>
<Folder size={16} />
Open Location Inspector
</button>
</PopoverSection>
</>
) : (
<PopoverSection>
<div className="px-2 py-1.5">
<div className="text-xs text-ink-dull mb-2">
Path is outside any location
</div>
<Button
size="sm"
variant="accent"
onClick={() => {
const initialPath =
"Physical" in path
? path.Physical.path
: undefined;
useAddStorageDialog(undefined, initialPath);
popover.setOpen(false);
}}
>
Add Location
</Button>
</div>
</div>
</PopoverSection>
<PopoverDivider />
<PopoverSection>
{!isIndexed && (
<button
onClick={async () => {
await enableIndexing.mutateAsync({
id: matchingLocation.id,
index_mode: "deep",
});
popover.setOpen(false);
}}
className="flex items-center gap-2 px-2 py-1.5 rounded-md text-xs font-medium text-ink hover:bg-app-hover transition-colors"
>
<Eye size={16} />
Enable Indexing
</button>
)}
<button
onClick={() => {
clearSelection();
popover.setOpen(false);
}}
className="flex items-center gap-2 px-2 py-1.5 rounded-md text-xs font-medium text-ink hover:bg-app-hover transition-colors"
>
<Folder size={16} />
Open Location Inspector
</button>
</PopoverSection>
</PopoverSection>
)}
</PopoverContainer>
</Popover>
</>
);
}
@@ -257,71 +302,69 @@ export function PathBar({ path, devices, onNavigate }: PathBarProps) {
"focus-within:bg-sidebar-box/30 focus-within:border-sidebar-line/40",
)}
>
<img
src={deviceIcon}
alt="Device"
className="size-5 opacity-60 flex-shrink-0"
/>
<img
src={deviceIcon}
alt="Device"
className="size-5 opacity-60 flex-shrink-0"
/>
{showUri ? (
<input
type="text"
value={uri}
readOnly
className={clsx(
"bg-transparent border-0 outline-none ring-0 flex-1 min-w-0",
"text-xs font-medium text-sidebar-ink",
"placeholder:text-sidebar-inkFaint",
"select-all cursor-text",
"focus:ring-0 focus:outline-none",
)}
placeholder="No path selected"
/>
) : isExpanded ? (
<div className="flex items-center gap-1 flex-1 min-w-0 overflow-hidden">
{segments.map((segment, index) => {
const isLast = index === segments.length - 1;
return (
<div
key={index}
className="flex items-center gap-1 flex-shrink-0"
>
<button
onClick={() =>
!isLast && onNavigate(segment.path)
}
disabled={isLast}
className={clsx(
"text-xs font-medium transition-colors whitespace-nowrap",
isLast
? "text-sidebar-ink cursor-default"
: "text-sidebar-inkDull hover:text-sidebar-ink cursor-pointer",
)}
{showUri ? (
<input
type="text"
value={uri}
readOnly
className={clsx(
"bg-transparent border-0 outline-none ring-0 flex-1 min-w-0",
"text-xs font-medium text-sidebar-ink",
"placeholder:text-sidebar-inkFaint",
"select-all cursor-text",
"focus:ring-0 focus:outline-none",
)}
placeholder="No path selected"
/>
) : isExpanded ? (
<div className="flex items-center gap-1 flex-1 min-w-0 overflow-hidden">
{segments.map((segment, index) => {
const isLast = index === segments.length - 1;
return (
<div
key={index}
className="flex items-center gap-1 flex-shrink-0"
>
{segment.name}
</button>
{!isLast && (
<CaretRight size={12} />
)}
</div>
);
})}
</div>
) : (
<input
type="text"
value={currentDir}
readOnly
className={clsx(
"bg-transparent border-0 outline-none ring-0 flex-1 min-w-0",
"text-xs font-medium text-sidebar-ink",
"placeholder:text-sidebar-inkFaint",
"select-all cursor-text",
"focus:ring-0 focus:outline-none",
)}
placeholder="No path selected"
/>
)}
<button
onClick={() =>
!isLast && onNavigate(segment.path)
}
disabled={isLast}
className={clsx(
"text-xs font-medium transition-colors whitespace-nowrap",
isLast
? "text-sidebar-ink cursor-default"
: "text-sidebar-inkDull hover:text-sidebar-ink cursor-pointer",
)}
>
{segment.name}
</button>
{!isLast && <CaretRight size={12} />}
</div>
);
})}
</div>
) : (
<input
type="text"
value={currentDir}
readOnly
className={clsx(
"bg-transparent border-0 outline-none ring-0 flex-1 min-w-0",
"text-xs font-medium text-sidebar-ink",
"placeholder:text-sidebar-inkFaint",
"select-all cursor-text",
"focus:ring-0 focus:outline-none",
)}
placeholder="No path selected"
/>
)}
</motion.div>
<IndexIndicator path={path} />
</div>