itsamejms
a69d7caefb
- chunk(): byte splits that landed mid multibyte char silently dropped the whole chunk (accented/CJK lore). Back the boundary off to a char boundary; regression test included. - add_document(): DELETE the source first — re-adding a file no longer doubles its chunks. - chunks now record their embed_model (ALTER TABLE migration for old DBs); search skips chunks from a different model; rag_list reports it; new rag_reindex re-embeds everything, surfaced in the Lore panel as a mismatch banner with a one-click Reindex.
414 lines
18 KiB
TypeScript
414 lines
18 KiB
TypeScript
import { useEffect, useState } from "react";
|
||
import { invoke } from "@tauri-apps/api/core";
|
||
import { open } from "@tauri-apps/plugin-dialog";
|
||
import { Trash2, ChevronDown, ChevronRight, FileUp, FolderOpen } from "lucide-react";
|
||
import { useToast } from "./Toast";
|
||
import { addToLore } from "../lib/lore";
|
||
|
||
interface RagSource {
|
||
source: string;
|
||
chunks: number;
|
||
model: string;
|
||
}
|
||
|
||
interface RagHit {
|
||
text: string;
|
||
source: string;
|
||
score: number;
|
||
}
|
||
|
||
interface RagChunk {
|
||
id: number;
|
||
preview: string;
|
||
}
|
||
|
||
export function LorePanel() {
|
||
const [source, setSource] = useState("");
|
||
const [text, setText] = useState("");
|
||
const [sources, setSources] = useState<RagSource[]>([]);
|
||
const [adding, setAdding] = useState(false);
|
||
const [addingDir, setAddingDir] = useState(false);
|
||
const [msg, setMsg] = useState("");
|
||
|
||
const [query, setQuery] = useState("");
|
||
const [hits, setHits] = useState<RagHit[]>([]);
|
||
const [searching, setSearching] = useState(false);
|
||
const [confirmClear, setConfirmClear] = useState(false);
|
||
// ponytail: source filter — multi-select chips. Empty set = search all.
|
||
const [filterSources, setFilterSources] = useState<Set<string>>(new Set());
|
||
// ponytail: chunk preview — expand a source to see its embedded chunks.
|
||
const [expanded, setExpanded] = useState<string | null>(null);
|
||
const [chunks, setChunks] = useState<RagChunk[]>([]);
|
||
const [loadingChunks, setLoadingChunks] = useState(false);
|
||
// ponytail: current embed model, to flag sources indexed with a different
|
||
// one (search skips those — garbage cosine scores). "" = still loading.
|
||
const [embedModel, setEmbedModel] = useState("");
|
||
const [reindexing, setReindexing] = useState(false);
|
||
const { addToast } = useToast();
|
||
|
||
const stale =
|
||
embedModel !== "" && sources.some((s) => s.model !== "" && s.model !== embedModel);
|
||
|
||
async function loadSources() {
|
||
try {
|
||
setSources(await invoke<RagSource[]>("rag_list"));
|
||
} catch (e) {
|
||
console.error(e);
|
||
}
|
||
}
|
||
|
||
useEffect(() => {
|
||
loadSources();
|
||
invoke<{ embed_model: string }>("get_llm_config")
|
||
.then((c) => setEmbedModel(c.embed_model))
|
||
.catch(() => {});
|
||
}, []);
|
||
|
||
async function loadChunks(s: string) {
|
||
setLoadingChunks(true);
|
||
try {
|
||
setChunks(await invoke<RagChunk[]>("rag_chunks", { source: s }));
|
||
} catch (e) {
|
||
console.error(e);
|
||
}
|
||
setLoadingChunks(false);
|
||
}
|
||
|
||
function toggleSource(s: string) {
|
||
setFilterSources((prev) => {
|
||
const next = new Set(prev);
|
||
if (next.has(s)) next.delete(s);
|
||
else next.add(s);
|
||
return next;
|
||
});
|
||
}
|
||
|
||
async function add() {
|
||
if (!source.trim() || !text.trim()) return;
|
||
setAdding(true);
|
||
setMsg("");
|
||
try {
|
||
const n = await invoke<number>("rag_add", { req: { source, text } });
|
||
setMsg(`Indexed ${n} chunk${n === 1 ? "" : "s"}`);
|
||
setText("");
|
||
loadSources();
|
||
} catch (e) {
|
||
setMsg(String(e));
|
||
}
|
||
setAdding(false);
|
||
setTimeout(() => setMsg(""), 3000);
|
||
}
|
||
|
||
async function clearOne(s: string) {
|
||
await invoke("rag_clear", { source: s });
|
||
if (expanded === s) setExpanded(null);
|
||
loadSources();
|
||
}
|
||
|
||
async function clearAll() {
|
||
await invoke("rag_clear", { source: null });
|
||
setConfirmClear(false);
|
||
setExpanded(null);
|
||
loadSources();
|
||
}
|
||
|
||
async function reindex() {
|
||
setReindexing(true);
|
||
try {
|
||
const r = await invoke<{ sources: number; chunks: number }>("rag_reindex");
|
||
addToast(`Reindexed ${r.sources} sources · ${r.chunks} chunks`, "success");
|
||
await loadSources();
|
||
} catch (e) {
|
||
addToast(`Reindex failed: ${e}`, "error");
|
||
}
|
||
setReindexing(false);
|
||
}
|
||
|
||
// ponytail: file upload via native HTML input — no tauri-plugin-dialog
|
||
// needed. Reads .md/.txt contents in the webview and indexes each file as
|
||
// its own lore source. Multiple files supported.
|
||
function onFiles(e: React.ChangeEvent<HTMLInputElement>) {
|
||
const files = Array.from(e.target.files ?? []);
|
||
e.target.value = "";
|
||
for (const f of files) {
|
||
const reader = new FileReader();
|
||
reader.onload = () => {
|
||
const text = String(reader.result ?? "");
|
||
addToLore(f.name, text).then(() => loadSources());
|
||
};
|
||
reader.onerror = () => addToast(`Failed to read ${f.name}`, "error");
|
||
reader.readAsText(f);
|
||
}
|
||
if (files.length) addToast(`Indexing ${files.length} file${files.length === 1 ? "" : "s"}…`, "info");
|
||
}
|
||
|
||
// ponytail: pick a folder and let Rust walk it (recursive, .md/.txt only).
|
||
// Doing the walk in Rust sidesteps the webview fs scope — an external
|
||
// world-bible folder needs no fs permission grant this way.
|
||
async function addDirectory() {
|
||
const selected = await open({ directory: true, multiple: false });
|
||
if (!selected || typeof selected !== "string") return;
|
||
setAddingDir(true);
|
||
try {
|
||
const report = await invoke<{ files: number; chunks: number; skipped: string[] }>(
|
||
"rag_add_directory",
|
||
{ req: { path: selected } },
|
||
);
|
||
await loadSources();
|
||
if (report.files === 0) {
|
||
addToast("No .md / .txt files found in that folder", "info");
|
||
} else {
|
||
addToast(
|
||
`Indexed ${report.files} file${report.files === 1 ? "" : "s"} · ${report.chunks} chunks`,
|
||
"success",
|
||
);
|
||
}
|
||
for (const s of report.skipped) console.warn("lore dir import skipped:", s);
|
||
} catch (e) {
|
||
addToast(`Folder import failed: ${e}`, "error");
|
||
}
|
||
setAddingDir(false);
|
||
}
|
||
|
||
async function search() {
|
||
if (!query.trim()) return;
|
||
setSearching(true);
|
||
try {
|
||
const all = await invoke<RagHit[]>("rag_search", { query, topK: 12 });
|
||
// ponytail: client-side source filter — the backend scans everything,
|
||
// we just hide hits from unselected sources. Fine at this corpus size.
|
||
setHits(filterSources.size === 0 ? all : all.filter((h) => filterSources.has(h.source)));
|
||
} catch (e) {
|
||
console.error(e);
|
||
}
|
||
setSearching(false);
|
||
}
|
||
|
||
return (
|
||
<div className="flex gap-4 h-full text-sm">
|
||
{/* Left pane: add + indexed sources */}
|
||
<div className="flex flex-col gap-3 w-1/2 min-w-0 overflow-y-auto">
|
||
<div className="flex flex-col gap-2">
|
||
<h3 className="font-heading text-[var(--color-gold-bright)] text-sm">Add to Lore</h3>
|
||
<input
|
||
className="rounded-lg bg-[var(--color-bg-surface)] border border-[var(--color-border-glass)] px-3 py-1.5 text-[var(--color-text-primary)] placeholder-[var(--color-text-dim)] focus:outline-none focus:border-[var(--color-gold-bright)] text-sm"
|
||
value={source}
|
||
onChange={(e) => setSource(e.target.value)}
|
||
placeholder="Source name (e.g. World Bible, Session 3 Notes)"
|
||
/>
|
||
<textarea
|
||
className="rounded-lg bg-[var(--color-bg-surface)] border border-[var(--color-border-glass)] px-3 py-2 text-[var(--color-text-primary)] placeholder-[var(--color-text-dim)] focus:outline-none focus:border-[var(--color-gold-bright)] text-sm min-h-40 resize-y"
|
||
value={text}
|
||
onChange={(e) => setText(e.target.value)}
|
||
placeholder="Paste lore text. It will be chunked on paragraphs, embedded via nomic-embed-text, and stored locally."
|
||
/>
|
||
<button
|
||
onClick={add}
|
||
disabled={adding}
|
||
className="rounded-lg bg-[var(--color-gold-bright)] text-[var(--color-bg-deep)] px-4 py-2 text-sm font-semibold hover:bg-[var(--color-gold-muted)] transition-colors cursor-pointer disabled:opacity-50"
|
||
>
|
||
{adding ? "Embedding…" : "Add to Lore"}
|
||
</button>
|
||
{/* ponytail: file upload — native input, no dialog plugin. */}
|
||
<label className="flex items-center gap-2 rounded-lg bg-[var(--color-bg-surface)] border border-[var(--color-border-glass)] px-3 py-2 text-sm text-[var(--color-text-dim)] hover:text-[var(--color-gold-bright)] hover:border-[var(--color-gold-bright)] transition-colors cursor-pointer">
|
||
<FileUp size={14} />
|
||
Upload .md / .txt
|
||
<input
|
||
type="file"
|
||
accept=".md,.txt,.markdown,text/plain,text/markdown"
|
||
multiple
|
||
className="hidden"
|
||
onChange={onFiles}
|
||
/>
|
||
</label>
|
||
{/* ponytail: directory import — walks every .md/.txt recursively in
|
||
Rust (no webview fs-scope needed for an external folder). */}
|
||
<button
|
||
onClick={addDirectory}
|
||
disabled={addingDir}
|
||
className="flex items-center gap-2 rounded-lg bg-[var(--color-bg-surface)] border border-[var(--color-border-glass)] px-3 py-2 text-sm text-[var(--color-text-dim)] hover:text-[var(--color-gold-bright)] hover:border-[var(--color-gold-bright)] transition-colors cursor-pointer disabled:opacity-50"
|
||
>
|
||
<FolderOpen size={14} />
|
||
{addingDir ? "Indexing…" : "Add folder…"}
|
||
</button>
|
||
{msg && <span className="text-xs text-[var(--color-text-secondary)]">{msg}</span>}
|
||
</div>
|
||
|
||
<div className="flex flex-col gap-2">
|
||
<div className="flex items-center justify-between">
|
||
<h3 className="font-heading text-[var(--color-gold-bright)] text-sm">Indexed Sources</h3>
|
||
{/* ponytail: embed-model mismatch banner — search silently skips
|
||
these chunks, so surface it with the one-click fix. */}
|
||
{stale && (
|
||
<div className="flex items-center gap-1.5">
|
||
<span className="text-[10px] text-[var(--color-gold-bright)]">Indexed with an old embedding model</span>
|
||
<button
|
||
onClick={reindex}
|
||
disabled={reindexing}
|
||
className="rounded bg-[var(--color-gold-bright)] text-[var(--color-bg-deep)] px-2 py-0.5 text-[10px] font-semibold cursor-pointer disabled:opacity-50"
|
||
>
|
||
{reindexing ? "Reindexing…" : "Reindex all"}
|
||
</button>
|
||
</div>
|
||
)}
|
||
{sources.length > 0 && (
|
||
confirmClear ? (
|
||
<div className="flex items-center gap-1">
|
||
<span className="text-[10px] text-[var(--color-danger)]">Clear all sources?</span>
|
||
<button
|
||
onClick={clearAll}
|
||
className="rounded bg-[var(--color-danger)] text-white px-2 py-0.5 text-[10px] font-semibold cursor-pointer"
|
||
>
|
||
Yes
|
||
</button>
|
||
<button
|
||
onClick={() => setConfirmClear(false)}
|
||
className="rounded bg-[var(--color-bg-surface)] border border-[var(--color-border-subtle)] text-[var(--color-text-dim)] px-2 py-0.5 text-[10px] cursor-pointer"
|
||
>
|
||
No
|
||
</button>
|
||
</div>
|
||
) : (
|
||
<button
|
||
onClick={() => setConfirmClear(true)}
|
||
className="text-xs text-[var(--color-text-dim)] hover:text-[var(--color-danger)] cursor-pointer transition-colors"
|
||
>
|
||
clear all
|
||
</button>
|
||
)
|
||
)}
|
||
</div>
|
||
{sources.length === 0 ? (
|
||
<p className="text-xs text-[var(--color-text-dim)]">No lore indexed yet.</p>
|
||
) : (
|
||
<ul className="flex flex-col gap-1">
|
||
{sources.map((s) => {
|
||
const isOpen = expanded === s.source;
|
||
return (
|
||
<li key={s.source} className="rounded-lg bg-[var(--color-bg-surface)] border border-[var(--color-border-glass)] overflow-hidden">
|
||
<div className="flex items-center gap-1.5 px-2 py-1.5">
|
||
<button
|
||
onClick={() => {
|
||
if (isOpen) { setExpanded(null); }
|
||
else { setExpanded(s.source); loadChunks(s.source); }
|
||
}}
|
||
className="text-[var(--color-text-dim)] hover:text-[var(--color-gold-bright)] cursor-pointer shrink-0"
|
||
aria-label={isOpen ? "Collapse chunks" : "Preview chunks"}
|
||
>
|
||
{isOpen ? <ChevronDown size={14} /> : <ChevronRight size={14} />}
|
||
</button>
|
||
<span className="text-[var(--color-text-primary)] text-xs truncate flex-1">
|
||
{s.source} <span className="text-[var(--color-text-dim)]">· {s.chunks} chunks</span>
|
||
</span>
|
||
<button
|
||
onClick={() => clearOne(s.source)}
|
||
className="text-[var(--color-text-dim)] hover:text-[var(--color-danger)] cursor-pointer transition-colors shrink-0"
|
||
title="Clear this source"
|
||
>
|
||
<Trash2 size={14} />
|
||
</button>
|
||
</div>
|
||
{isOpen && (
|
||
<div className="px-2 pb-2 border-t border-[var(--color-border-subtle)] pt-1.5">
|
||
{loadingChunks ? (
|
||
<span className="text-[10px] text-[var(--color-text-dim)]">Loading chunks…</span>
|
||
) : chunks.length === 0 ? (
|
||
<span className="text-[10px] text-[var(--color-text-dim)]">No chunks.</span>
|
||
) : (
|
||
<div className="flex flex-col gap-1 max-h-40 overflow-y-auto">
|
||
{chunks.map((c) => (
|
||
<div key={c.id} className="rounded bg-[var(--color-bg-deep)] border border-[var(--color-border-subtle)] px-2 py-1 text-[10px] text-[var(--color-text-secondary)] leading-relaxed">
|
||
<span className="text-[var(--color-text-dim)] font-mono">#{c.id}</span>{" "}
|
||
{c.preview}
|
||
{c.preview.length >= 200 ? "…" : ""}
|
||
</div>
|
||
))}
|
||
</div>
|
||
)}
|
||
</div>
|
||
)}
|
||
</li>
|
||
);
|
||
})}
|
||
</ul>
|
||
)}
|
||
</div>
|
||
</div>
|
||
|
||
{/* Right pane: search + explore */}
|
||
<div className="flex flex-col gap-2 w-1/2 min-w-0 border-l border-[var(--color-border-subtle)] pl-4">
|
||
<h3 className="font-heading text-[var(--color-gold-bright)] text-sm">Search Lore</h3>
|
||
<div className="flex gap-2">
|
||
<input
|
||
className="flex-1 min-w-0 rounded-lg bg-[var(--color-bg-surface)] border border-[var(--color-border-glass)] px-3 py-1.5 text-[var(--color-text-primary)] placeholder-[var(--color-text-dim)] focus:outline-none focus:border-[var(--color-gold-bright)] text-sm"
|
||
value={query}
|
||
onChange={(e) => setQuery(e.target.value)}
|
||
onKeyDown={(e) => e.key === "Enter" && search()}
|
||
placeholder="Ask something your lore should answer…"
|
||
/>
|
||
<button
|
||
onClick={search}
|
||
disabled={searching}
|
||
className="rounded-lg bg-[var(--color-gold-bright)] text-[var(--color-bg-deep)] px-3 py-1.5 text-sm font-semibold hover:bg-[var(--color-gold-muted)] transition-colors cursor-pointer disabled:opacity-50"
|
||
>
|
||
{searching ? "…" : "Search"}
|
||
</button>
|
||
</div>
|
||
{/* Source filter chips */}
|
||
{sources.length > 0 && (
|
||
<div className="flex flex-wrap gap-1">
|
||
<span className="text-[10px] text-[var(--color-text-dim)] self-center">Filter:</span>
|
||
<button
|
||
onClick={() => setFilterSources(new Set())}
|
||
className={`rounded-full px-2 py-0.5 text-[10px] border cursor-pointer transition-colors ${
|
||
filterSources.size === 0
|
||
? "bg-[var(--color-gold-glow)] border-[var(--color-gold-bright)] text-[var(--color-gold-bright)]"
|
||
: "bg-[var(--color-bg-deep)] border-[var(--color-border-subtle)] text-[var(--color-text-dim)] hover:text-[var(--color-text-secondary)]"
|
||
}`}
|
||
>
|
||
all
|
||
</button>
|
||
{sources.map((s) => (
|
||
<button
|
||
key={s.source}
|
||
onClick={() => toggleSource(s.source)}
|
||
className={`rounded-full px-2 py-0.5 text-[10px] border cursor-pointer transition-colors truncate max-w-32 ${
|
||
filterSources.has(s.source)
|
||
? "bg-[var(--color-gold-glow)] border-[var(--color-gold-bright)] text-[var(--color-gold-bright)]"
|
||
: "bg-[var(--color-bg-deep)] border-[var(--color-border-subtle)] text-[var(--color-text-dim)] hover:text-[var(--color-text-secondary)]"
|
||
}`}
|
||
>
|
||
{s.source}
|
||
</button>
|
||
))}
|
||
</div>
|
||
)}
|
||
<div className="flex-1 overflow-y-auto">
|
||
{hits.length > 0 ? (
|
||
<ul className="flex flex-col gap-2">
|
||
{hits.map((h, i) => (
|
||
<li key={i} className="rounded-lg bg-[var(--color-bg-surface)] border border-[var(--color-border-glass)] p-2">
|
||
<div className="flex items-center justify-between mb-1">
|
||
<span className="text-[10px] text-[var(--color-gold-bright)]">{h.source}</span>
|
||
{/* ponytail: relevance bar instead of a raw cosine score a
|
||
DM can't read. Clamp 0–1, gold fill. */}
|
||
<div className="flex items-center gap-1.5">
|
||
<div className="w-12 h-1 rounded-full bg-[var(--color-bg-deep)] overflow-hidden">
|
||
<div className="h-full bg-[var(--color-gold-bright)]" style={{ width: `${Math.round(Math.max(0, Math.min(1, h.score)) * 100)}%` }} />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p className="text-xs text-[var(--color-text-primary)] leading-relaxed whitespace-pre-wrap">{h.text}</p>
|
||
</li>
|
||
))}
|
||
</ul>
|
||
) : (
|
||
<p className="text-xs text-[var(--color-text-dim)] italic mt-2">
|
||
{query ? "No matches." : "Search to retrieve the most relevant lore chunks."}
|
||
</p>
|
||
)}
|
||
</div>
|
||
</div>
|
||
</div>
|
||
);
|
||
} |