fix: RAG — UTF-8 chunk boundary, source dedupe, embed-model guard + reindex

- chunk(): byte splits that landed mid multibyte char silently dropped
  the whole chunk (accented/CJK lore). Back the boundary off to a char
  boundary; regression test included.
- add_document(): DELETE the source first — re-adding a file no longer
  doubles its chunks.
- chunks now record their embed_model (ALTER TABLE migration for old
  DBs); search skips chunks from a different model; rag_list reports it;
  new rag_reindex re-embeds everything, surfaced in the Lore panel as a
  mismatch banner with a one-click Reindex.
This commit is contained in:
itsamejms
2026-09-06 23:25:22 +01:00
parent 913345a05e
commit a69d7caefb
4 changed files with 159 additions and 22 deletions
+37
View File
@@ -8,6 +8,7 @@ import { addToLore } from "../lib/lore";
interface RagSource {
source: string;
chunks: number;
model: string;
}
interface RagHit {
@@ -39,8 +40,15 @@ export function LorePanel() {
const [expanded, setExpanded] = useState<string | null>(null);
const [chunks, setChunks] = useState<RagChunk[]>([]);
const [loadingChunks, setLoadingChunks] = useState(false);
// ponytail: current embed model, to flag sources indexed with a different
// one (search skips those — garbage cosine scores). "" = still loading.
const [embedModel, setEmbedModel] = useState("");
const [reindexing, setReindexing] = useState(false);
const { addToast } = useToast();
const stale =
embedModel !== "" && sources.some((s) => s.model !== "" && s.model !== embedModel);
async function loadSources() {
try {
setSources(await invoke<RagSource[]>("rag_list"));
@@ -51,6 +59,9 @@ export function LorePanel() {
useEffect(() => {
loadSources();
invoke<{ embed_model: string }>("get_llm_config")
.then((c) => setEmbedModel(c.embed_model))
.catch(() => {});
}, []);
async function loadChunks(s: string) {
@@ -101,6 +112,18 @@ export function LorePanel() {
loadSources();
}
async function reindex() {
setReindexing(true);
try {
const r = await invoke<{ sources: number; chunks: number }>("rag_reindex");
addToast(`Reindexed ${r.sources} sources · ${r.chunks} chunks`, "success");
await loadSources();
} catch (e) {
addToast(`Reindex failed: ${e}`, "error");
}
setReindexing(false);
}
// ponytail: file upload via native HTML input — no tauri-plugin-dialog
// needed. Reads .md/.txt contents in the webview and indexes each file as
// its own lore source. Multiple files supported.
@@ -214,6 +237,20 @@ export function LorePanel() {
<div className="flex flex-col gap-2">
<div className="flex items-center justify-between">
<h3 className="font-heading text-[var(--color-gold-bright)] text-sm">Indexed Sources</h3>
{/* ponytail: embed-model mismatch banner — search silently skips
these chunks, so surface it with the one-click fix. */}
{stale && (
<div className="flex items-center gap-1.5">
<span className="text-[10px] text-[var(--color-gold-bright)]">Indexed with an old embedding model</span>
<button
onClick={reindex}
disabled={reindexing}
className="rounded bg-[var(--color-gold-bright)] text-[var(--color-bg-deep)] px-2 py-0.5 text-[10px] font-semibold cursor-pointer disabled:opacity-50"
>
{reindexing ? "Reindexing…" : "Reindex all"}
</button>
</div>
)}
{sources.length > 0 && (
confirmClear ? (
<div className="flex items-center gap-1">