fix: RAG — UTF-8 chunk boundary, source dedupe, embed-model guard + reindex
- chunk(): byte splits that landed mid multibyte char silently dropped the whole chunk (accented/CJK lore). Back the boundary off to a char boundary; regression test included. - add_document(): DELETE the source first — re-adding a file no longer doubles its chunks. - chunks now record their embed_model (ALTER TABLE migration for old DBs); search skips chunks from a different model; rag_list reports it; new rag_reindex re-embeds everything, surfaced in the Lore panel as a mismatch banner with a one-click Reindex.
This commit is contained in:
@@ -8,6 +8,7 @@ import { addToLore } from "../lib/lore";
|
||||
interface RagSource {
|
||||
source: string;
|
||||
chunks: number;
|
||||
model: string;
|
||||
}
|
||||
|
||||
interface RagHit {
|
||||
@@ -39,8 +40,15 @@ export function LorePanel() {
|
||||
const [expanded, setExpanded] = useState<string | null>(null);
|
||||
const [chunks, setChunks] = useState<RagChunk[]>([]);
|
||||
const [loadingChunks, setLoadingChunks] = useState(false);
|
||||
// ponytail: current embed model, to flag sources indexed with a different
|
||||
// one (search skips those — garbage cosine scores). "" = still loading.
|
||||
const [embedModel, setEmbedModel] = useState("");
|
||||
const [reindexing, setReindexing] = useState(false);
|
||||
const { addToast } = useToast();
|
||||
|
||||
const stale =
|
||||
embedModel !== "" && sources.some((s) => s.model !== "" && s.model !== embedModel);
|
||||
|
||||
async function loadSources() {
|
||||
try {
|
||||
setSources(await invoke<RagSource[]>("rag_list"));
|
||||
@@ -51,6 +59,9 @@ export function LorePanel() {
|
||||
|
||||
useEffect(() => {
|
||||
loadSources();
|
||||
invoke<{ embed_model: string }>("get_llm_config")
|
||||
.then((c) => setEmbedModel(c.embed_model))
|
||||
.catch(() => {});
|
||||
}, []);
|
||||
|
||||
async function loadChunks(s: string) {
|
||||
@@ -101,6 +112,18 @@ export function LorePanel() {
|
||||
loadSources();
|
||||
}
|
||||
|
||||
async function reindex() {
|
||||
setReindexing(true);
|
||||
try {
|
||||
const r = await invoke<{ sources: number; chunks: number }>("rag_reindex");
|
||||
addToast(`Reindexed ${r.sources} sources · ${r.chunks} chunks`, "success");
|
||||
await loadSources();
|
||||
} catch (e) {
|
||||
addToast(`Reindex failed: ${e}`, "error");
|
||||
}
|
||||
setReindexing(false);
|
||||
}
|
||||
|
||||
// ponytail: file upload via native HTML input — no tauri-plugin-dialog
|
||||
// needed. Reads .md/.txt contents in the webview and indexes each file as
|
||||
// its own lore source. Multiple files supported.
|
||||
@@ -214,6 +237,20 @@ export function LorePanel() {
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<h3 className="font-heading text-[var(--color-gold-bright)] text-sm">Indexed Sources</h3>
|
||||
{/* ponytail: embed-model mismatch banner — search silently skips
|
||||
these chunks, so surface it with the one-click fix. */}
|
||||
{stale && (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="text-[10px] text-[var(--color-gold-bright)]">Indexed with an old embedding model</span>
|
||||
<button
|
||||
onClick={reindex}
|
||||
disabled={reindexing}
|
||||
className="rounded bg-[var(--color-gold-bright)] text-[var(--color-bg-deep)] px-2 py-0.5 text-[10px] font-semibold cursor-pointer disabled:opacity-50"
|
||||
>
|
||||
{reindexing ? "Reindexing…" : "Reindex all"}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
{sources.length > 0 && (
|
||||
confirmClear ? (
|
||||
<div className="flex items-center gap-1">
|
||||
|
||||
Reference in New Issue
Block a user