finally getting a dev version functioning locally... what a mess that was
This commit is contained in:
itsamejms
@@ -0,0 +1,26 @@
|
||||
(async () => {
|
||||
try {
|
||||
// In dev, load a local .env so ENV values like GEMINI_API_KEY are available to the main process.
|
||||
try {
|
||||
// eslint-disable-next-line no-var
|
||||
var dotenv = await import('dotenv');
|
||||
dotenv.config();
|
||||
console.log('[electron-main.cjs] Loaded .env into process.env');
|
||||
} catch (e) {
|
||||
// Not fatal; dotenv may not be installed in some environments
|
||||
console.log('[electron-main.cjs] dotenv not available, skipping .env load');
|
||||
}
|
||||
const mod = await import('./electron-main.js');
|
||||
const { ipcMain } = await import('electron');
|
||||
|
||||
ipcMain.on('renderer-log', (event, { level, msg }) => {
|
||||
const prefix = `[renderer ${level}]`;
|
||||
if (level === 'error') console.error(prefix, msg);
|
||||
else if (level === 'warn') console.warn(prefix, msg);
|
||||
else console.log(prefix, msg);
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Failed to load ESM electron main:', e);
|
||||
process.exit(1);
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,103 @@
|
||||
import { app, BrowserWindow, ipcMain } from 'electron';
|
||||
import path from 'path';
|
||||
import url from 'url';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// Keep an in-memory API key for the running session only. Renderer should still store key in localStorage.
|
||||
let inMemoryKey = null;
|
||||
|
||||
function createWindow() {
|
||||
// Resolve dirname equivalent in ESM
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const win = new BrowserWindow({
|
||||
width: 1200,
|
||||
height: 800,
|
||||
webPreferences: {
|
||||
// Preload is colocated with the electron main files after the refactor
|
||||
preload: path.join(__dirname, 'electron-preload.js'),
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
}
|
||||
});
|
||||
|
||||
const startUrl = process.env.ELECTRON_START_URL || url.pathToFileURL(path.join(process.cwd(), 'dist', 'index.html')).toString();
|
||||
win.loadURL(startUrl).catch(err => {
|
||||
console.error('[electron-main] Failed to load URL:', err);
|
||||
});
|
||||
|
||||
if (process.env.ELECTRON_START_URL) {
|
||||
try {
|
||||
const ses = win.webContents.session;
|
||||
ses.webRequest.onHeadersReceived((details, callback) => {
|
||||
const headers = details.responseHeaders || {};
|
||||
// Allow jsDelivr CDN for scripts used by third-party libs (dev only)
|
||||
// Dev CSP: allow inline scripts, unsafe-eval and jsDelivr CDN for third-party libs (HMR and wasm loaders need inline scripts/eval)
|
||||
headers['Content-Security-Policy'] = [
|
||||
"default-src 'self' 'unsafe-eval' 'unsafe-inline' data:; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.jsdelivr.net; img-src 'self' data:; connect-src *; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' data: https://fonts.gstatic.com"
|
||||
];
|
||||
callback({ responseHeaders: headers });
|
||||
});
|
||||
} catch (e) {
|
||||
console.warn('[electron-main] Failed to inject dev CSP:', e);
|
||||
}
|
||||
win.webContents.once('did-frame-finish-load', () => {
|
||||
try {
|
||||
win.webContents.openDevTools({ mode: 'right' });
|
||||
} catch (e) {
|
||||
console.warn('[electron-main] Could not open DevTools:', e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
win.webContents.on('did-finish-load', () => {
|
||||
console.log('[electron-main] Renderer finished load; title=', win.getTitle());
|
||||
});
|
||||
}
|
||||
|
||||
app.whenReady().then(() => {
|
||||
createWindow();
|
||||
|
||||
app.on('activate', function () {
|
||||
if (BrowserWindow.getAllWindows().length === 0) createWindow();
|
||||
});
|
||||
});
|
||||
|
||||
app.on('window-all-closed', function () {
|
||||
if (process.platform !== 'darwin') app.quit();
|
||||
});
|
||||
|
||||
ipcMain.handle('generate-avatar', async (event, prompt) => {
|
||||
try {
|
||||
console.log('[electron-main] generate-avatar handler invoked');
|
||||
console.log('[electron-main] prompt length:', (prompt || '').length);
|
||||
|
||||
// Prefer an in-memory key, then environment variables
|
||||
const apiKey = inMemoryKey || process.env.GEMINI_API_KEY || process.env.API_KEY;
|
||||
console.log('[electron-main] apiKey present?', !!apiKey, '(will prefer GEMINI_API_KEY if set)');
|
||||
|
||||
if (apiKey) {
|
||||
try {
|
||||
console.log('[electron-main] Calling GenAI helper...');
|
||||
const imageData = await generateAvatarWithGenAI(prompt || '', apiKey);
|
||||
console.log('[electron-main] GenAI helper returned image, length:', imageData?.length || 0);
|
||||
return { image: imageData };
|
||||
} catch (e) {
|
||||
console.error('[electron-main] GenAI generation failed:', e?.message || e);
|
||||
}
|
||||
} else {
|
||||
console.log('[electron-main] No API key present — skipping GenAI call and returning placeholder');
|
||||
}
|
||||
} catch (outerErr) {
|
||||
console.error('[electron-main] Unexpected error in generate-avatar handler:', outerErr);
|
||||
}
|
||||
|
||||
// Fallback placeholder image
|
||||
const placeholder = `data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='1400' height='900'><rect width='100%' height='100%' fill='%230f172a' /><text x='50%' y='50%' fill='white' font-size='40' font-family='Inter' dominant-baseline='middle' text-anchor='middle'>Placeholder: ${encodeURIComponent((prompt||'').substring(0,80))}</text></svg>`;
|
||||
console.log('[electron-main] Returning placeholder image (length):', placeholder.length);
|
||||
return { image: placeholder };
|
||||
});
|
||||
|
||||
// Expose simple key management for the renderer via ipc (session-only)
|
||||
// No renderer-side key persistence — frontend uses localStorage exclusively now.
|
||||
@@ -0,0 +1,45 @@
|
||||
const { contextBridge, ipcRenderer } = require('electron');
|
||||
|
||||
contextBridge.exposeInMainWorld('electronAPI', {
|
||||
// Accept an optional apiKey argument so the renderer can pass a stored key from localStorage
|
||||
generateAvatar: (prompt, apiKey) => ipcRenderer.invoke('generate-avatar', prompt, apiKey)
|
||||
});
|
||||
|
||||
|
||||
contextBridge.exposeInMainWorld('electronLog', {
|
||||
info: (msg) => ipcRenderer.send('renderer-log', { level: 'info', msg }),
|
||||
warn: (msg) => ipcRenderer.send('renderer-log', { level: 'warn', msg }),
|
||||
error: (msg) => ipcRenderer.send('renderer-log', { level: 'error', msg }),
|
||||
});
|
||||
|
||||
try {
|
||||
ipcRenderer.send('renderer-log', { level: 'info', msg: 'preload loaded' });
|
||||
} catch (e) {
|
||||
}
|
||||
|
||||
try {
|
||||
const forward = (level, args) => {
|
||||
try {
|
||||
const msg = args.map(a => {
|
||||
try { return typeof a === 'string' ? a : JSON.stringify(a); } catch (e) { return String(a); }
|
||||
}).join(' ');
|
||||
ipcRenderer.send('renderer-log', { level, msg });
|
||||
} catch (e) { }
|
||||
};
|
||||
|
||||
const origError = console.error.bind(console);
|
||||
console.error = (...args) => { forward('error', args); origError(...args); };
|
||||
|
||||
const origWarn = console.warn.bind(console);
|
||||
console.warn = (...args) => { forward('warn', args); origWarn(...args); };
|
||||
|
||||
const origLog = console.log.bind(console);
|
||||
console.log = (...args) => { forward('info', args); origLog(...args); };
|
||||
|
||||
// DOM snapshot logging removed — keep console forwarding only.
|
||||
|
||||
window.addEventListener('error', (ev) => {
|
||||
try { ipcRenderer.send('renderer-log', { level: 'error', msg: `window.onerror: ${ev.message} ${ev.filename}:${ev.lineno}:${ev.colno}` }); } catch (e) {}
|
||||
});
|
||||
} catch (e) {
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
import React, { useState } from 'react';
|
||||
import { AppState, AvatarConfig, Rect } from '../shared/types';
|
||||
import AvatarCreator from './components/AvatarCreator';
|
||||
import RiggingEditor from './components/RiggingEditor';
|
||||
import Studio from './components/Studio';
|
||||
|
||||
const App: React.FC = () => {
|
||||
const [appState, setAppState] = useState<AppState>(AppState.SETUP);
|
||||
const [generatedData, setGeneratedData] = useState<{url: string, name: string, initialData?: any} | null>(null);
|
||||
const [avatar, setAvatar] = useState<AvatarConfig | null>(null);
|
||||
|
||||
const handleStartCreation = async () => {
|
||||
try {
|
||||
if ((window as any).electronLog) (window as any).electronLog.info('handleStartCreation called');
|
||||
// No blocking API key flow — we just try to ensure a key is set for better UX
|
||||
setAppState(AppState.CREATION);
|
||||
} catch (error) {
|
||||
console.error("Error during API key selection:", error);
|
||||
if ((window as any).electronLog) (window as any).electronLog.error(`API key selection failed: ${String(error)}`);
|
||||
setAppState(AppState.CREATION);
|
||||
}
|
||||
};
|
||||
|
||||
const [hasKey, setHasKey] = useState<boolean>(false);
|
||||
|
||||
const [showKeyModal, setShowKeyModal] = React.useState(false);
|
||||
const [keyInput, setKeyInput] = React.useState('');
|
||||
|
||||
const refreshKeyStatus = async () => {
|
||||
try {
|
||||
const present = !!(typeof window !== 'undefined' && localStorage.getItem('GEMINI_API_KEY'));
|
||||
setHasKey(present);
|
||||
} catch (e) {
|
||||
console.warn('Failed to check API key status from localStorage', e);
|
||||
setHasKey(false);
|
||||
}
|
||||
};
|
||||
|
||||
React.useEffect(() => {
|
||||
refreshKeyStatus();
|
||||
}, []);
|
||||
|
||||
const openKeyModal = () => {
|
||||
setKeyInput('');
|
||||
setShowKeyModal(true);
|
||||
};
|
||||
|
||||
const submitKey = async () => {
|
||||
try {
|
||||
if (!keyInput) return;
|
||||
// Only store in localStorage (renderer-side). Do not attempt IPC or disk persistence.
|
||||
try { localStorage.setItem('GEMINI_API_KEY', keyInput); } catch (e) { console.warn('Failed to write key to localStorage', e); alert('Failed to save key to localStorage'); return; }
|
||||
setShowKeyModal(false);
|
||||
refreshKeyStatus();
|
||||
try { window.electronLog?.info('API key saved to localStorage'); } catch {}
|
||||
alert('API key saved to localStorage');
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
alert('Error saving key');
|
||||
}
|
||||
};
|
||||
|
||||
const handleClearKey = async () => {
|
||||
try {
|
||||
try { localStorage.removeItem('GEMINI_API_KEY'); } catch (e) { console.warn('Failed to remove key from localStorage', e); alert('Failed to clear key from localStorage'); return; }
|
||||
refreshKeyStatus();
|
||||
alert('API key cleared from localStorage');
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
};
|
||||
|
||||
const handleAvatarGenerated = (url: string, name: string, initialData?: any) => {
|
||||
if ((window as any).electronLog) (window as any).electronLog.info(`avatar generated: ${name}`);
|
||||
setGeneratedData({ url, name, initialData });
|
||||
setAppState(AppState.RIGGING);
|
||||
};
|
||||
|
||||
const handleRiggingComplete = (data: {
|
||||
leftEye: Rect, rightEye: Rect, mouth: Rect, skinColor: string,
|
||||
textureClosedEye: Rect, textureOpenMouth: Rect,
|
||||
mainBody: Rect, chromaKeyColor: string
|
||||
}) => {
|
||||
if (generatedData) {
|
||||
setAvatar({
|
||||
imageUrl: generatedData.url,
|
||||
name: generatedData.name,
|
||||
description: '',
|
||||
leftEye: data.leftEye,
|
||||
rightEye: data.rightEye,
|
||||
mouth: data.mouth,
|
||||
skinColor: data.skinColor,
|
||||
textureClosedEye: data.textureClosedEye,
|
||||
textureOpenMouth: data.textureOpenMouth,
|
||||
mainBody: data.mainBody,
|
||||
chromaKeyColor: data.chromaKeyColor
|
||||
});
|
||||
setAppState(AppState.STUDIO);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="min-h-screen bg-slate-900 text-white">
|
||||
{appState === AppState.SETUP && (
|
||||
<div className="container mx-auto px-4 py-12 flex flex-col items-center justify-center min-h-screen">
|
||||
<div className="text-center mb-12 space-y-4">
|
||||
<h1 className="text-6xl font-bold text-transparent bg-clip-text bg-gradient-to-r from-cyan-400 via-blue-500 to-purple-600 brand-font tracking-tighter">
|
||||
GEMINI V-STUDIO
|
||||
</h1>
|
||||
<p className="text-xl text-slate-400 max-w-2xl mx-auto">
|
||||
The next-generation browser-based VTuber studio. Generate your persona with AI and animate it with your face.
|
||||
</p>
|
||||
<button
|
||||
onClick={handleStartCreation}
|
||||
className="mt-8 px-8 py-4 bg-white text-slate-900 rounded-full font-bold hover:bg-cyan-50 transition-colors shadow-[0_0_20px_rgba(255,255,255,0.3)]"
|
||||
>
|
||||
Start Creation
|
||||
</button>
|
||||
<div className="mt-4 flex items-center justify-center gap-3">
|
||||
<button onClick={openKeyModal} className="px-4 py-2 rounded bg-slate-700 hover:bg-slate-600 text-sm">Set API Key</button>
|
||||
<button onClick={handleClearKey} className="px-4 py-2 rounded bg-red-600 hover:bg-red-500 text-sm">Clear Key</button>
|
||||
<div className="text-sm text-slate-400">Key: {hasKey ? <span className="text-green-400">Saved</span> : <span className="text-rose-400">Not set</span>}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-8 w-full max-w-5xl">
|
||||
<div className="p-6 bg-slate-800/50 rounded-xl border border-slate-700 backdrop-blur-sm">
|
||||
<div className="h-12 w-12 bg-cyan-500/10 rounded-lg flex items-center justify-center mb-4 text-2xl">✨</div>
|
||||
<h3 className="text-xl font-bold mb-2">AI Generation</h3>
|
||||
<p className="text-slate-400">Describe your dream character. Gemini 3 Pro creates high-fidelity sprites in seconds.</p>
|
||||
</div>
|
||||
<div className="p-6 bg-slate-800/50 rounded-xl border border-slate-700 backdrop-blur-sm">
|
||||
<div className="h-12 w-12 bg-purple-500/10 rounded-lg flex items-center justify-center mb-4 text-2xl">📸</div>
|
||||
<h3 className="text-xl font-bold mb-2">Face Tracking</h3>
|
||||
<p className="text-slate-400">Powered by MediaPipe. No expensive equipment needed—just your webcam.</p>
|
||||
</div>
|
||||
<div className="p-6 bg-slate-800/50 rounded-xl border border-slate-700 backdrop-blur-sm">
|
||||
<div className="h-12 w-12 bg-pink-500/10 rounded-lg flex items-center justify-center mb-4 text-2xl">🎥</div>
|
||||
<h3 className="text-xl font-bold mb-2">Live Animation</h3>
|
||||
<p className="text-slate-400">Your avatar mimics your head movements and speech in real-time.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{appState === AppState.CREATION && (
|
||||
<div className="container mx-auto px-4 py-12 min-h-screen flex flex-col">
|
||||
<button
|
||||
onClick={() => setAppState(AppState.SETUP)}
|
||||
className="self-start mb-8 px-4 py-2 text-slate-400 hover:text-white transition-colors"
|
||||
>
|
||||
← Back to Home
|
||||
</button>
|
||||
<div className="flex-1 flex items-center justify-center">
|
||||
<AvatarCreator onAvatarGenerated={handleAvatarGenerated} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{appState === AppState.RIGGING && generatedData && (
|
||||
<div className="container mx-auto px-4 py-8 min-h-screen flex flex-col">
|
||||
<button
|
||||
onClick={() => setAppState(AppState.CREATION)}
|
||||
className="self-start mb-4 px-4 py-2 text-slate-400 hover:text-white transition-colors"
|
||||
>
|
||||
← Back to Generator
|
||||
</button>
|
||||
<RiggingEditor
|
||||
imageUrl={generatedData.url}
|
||||
initialData={generatedData.initialData}
|
||||
onComplete={handleRiggingComplete}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{appState === AppState.STUDIO && avatar && (
|
||||
<Studio
|
||||
avatar={avatar}
|
||||
onBack={() => setAppState(AppState.SETUP)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* TailwindDebug removed */}
|
||||
</div>
|
||||
{showKeyModal && (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60">
|
||||
<div className="bg-slate-900 rounded-xl p-6 w-full max-w-lg">
|
||||
<h3 className="text-lg font-bold mb-3">Enter Gemini API Key</h3>
|
||||
<p className="text-sm text-slate-400 mb-4">This key will be stored locally in the app data directory.</p>
|
||||
<input
|
||||
type="password"
|
||||
value={keyInput}
|
||||
onChange={(e) => setKeyInput(e.target.value)}
|
||||
className="w-full bg-slate-800 border border-slate-700 rounded px-3 py-2 text-white mb-4"
|
||||
placeholder="sk_..."
|
||||
/>
|
||||
<div className="flex justify-end gap-3">
|
||||
<button onClick={() => setShowKeyModal(false)} className="px-4 py-2 rounded bg-slate-700 hover:bg-slate-600">Cancel</button>
|
||||
<button onClick={submitKey} className="px-4 py-2 rounded bg-cyan-500 hover:bg-cyan-400 text-slate-900 font-bold">Save</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default App;
|
||||
@@ -0,0 +1,251 @@
|
||||
import React, { useState } from 'react';
|
||||
import { analyzeAvatarImage } from '../services/visionService';
|
||||
import { stitchAssets, fileToDataUrl } from '../services/imageService';
|
||||
import { generateAvatarImage } from '../services/geminiService';
|
||||
import LoadingSpinner from './LoadingSpinner';
|
||||
import { Rect } from '../../shared/types';
|
||||
|
||||
const placeholderGenerate = async (prompt: string) => {
|
||||
const text = encodeURIComponent((prompt || '').substring(0, 80));
|
||||
return `data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='1400' height='900'><rect width='100%' height='100%' fill='%230f172a' /><text x='50%' y='50%' fill='white' font-size='40' font-family='Inter' dominant-baseline='middle' text-anchor='middle'>Placeholder: ${text}</text></svg>`;
|
||||
};
|
||||
|
||||
interface AvatarCreatorProps {
|
||||
onAvatarGenerated: (url: string, name: string, initialData?: {
|
||||
leftEye?: Rect, rightEye?: Rect, mouth?: Rect, skinColor?: string,
|
||||
mainBody?: Rect, textureClosedEye?: Rect, textureOpenMouth?: Rect
|
||||
}) => void;
|
||||
}
|
||||
|
||||
const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
|
||||
const [mode, setMode] = useState<'generate' | 'upload'>('generate');
|
||||
|
||||
const [prompt, setPrompt] = useState('');
|
||||
const [name, setName] = useState('');
|
||||
const [status, setStatus] = useState<'idle' | 'generating' | 'analyzing' | 'stitching'>('idle');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const [baseFile, setBaseFile] = useState<File | null>(null);
|
||||
const [blinkFile, setBlinkFile] = useState<File | null>(null);
|
||||
const [talkFile, setTalkFile] = useState<File | null>(null);
|
||||
|
||||
const handleGenerate = async () => {
|
||||
if (!prompt || !name) return;
|
||||
|
||||
setStatus('generating');
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const imageUrl = await generateAvatarImage(prompt);
|
||||
|
||||
setStatus('analyzing');
|
||||
const analysisData = await analyzeAvatarImage(imageUrl);
|
||||
|
||||
if (analysisData) {
|
||||
onAvatarGenerated(imageUrl, name, analysisData);
|
||||
} else {
|
||||
onAvatarGenerated(imageUrl, name);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
setError("Failed to generate avatar. Please try again.");
|
||||
} finally {
|
||||
setStatus('idle');
|
||||
}
|
||||
};
|
||||
|
||||
const handleUpload = async () => {
|
||||
if (!baseFile || !name) return;
|
||||
|
||||
setStatus('stitching');
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const baseDataUrl = await fileToDataUrl(baseFile);
|
||||
const baseAnalysis = await analyzeAvatarImage(baseDataUrl);
|
||||
|
||||
let blinkDataUrl, blinkAnalysis;
|
||||
if (blinkFile) {
|
||||
blinkDataUrl = await fileToDataUrl(blinkFile);
|
||||
blinkAnalysis = await analyzeAvatarImage(blinkDataUrl);
|
||||
}
|
||||
|
||||
let talkDataUrl, talkAnalysis;
|
||||
if (talkFile) {
|
||||
talkDataUrl = await fileToDataUrl(talkFile);
|
||||
talkAnalysis = await analyzeAvatarImage(talkDataUrl);
|
||||
}
|
||||
|
||||
const { imageUrl, mainBody, textureClosedEye: stitchBlinkRect, textureOpenMouth: stitchTalkRect } = await stitchAssets(baseDataUrl, blinkDataUrl, talkDataUrl);
|
||||
|
||||
const mapRect = (r: Rect, container: Rect) => ({
|
||||
x: container.x + r.x * container.w,
|
||||
y: container.y + r.y * container.h,
|
||||
w: r.w * container.w,
|
||||
h: r.h * container.h
|
||||
});
|
||||
|
||||
let initialData: any = {
|
||||
mainBody,
|
||||
textureClosedEye: stitchBlinkRect,
|
||||
textureOpenMouth: stitchTalkRect
|
||||
};
|
||||
|
||||
if (baseAnalysis) {
|
||||
initialData.leftEye = mapRect(baseAnalysis.leftEye, mainBody);
|
||||
initialData.rightEye = mapRect(baseAnalysis.rightEye, mainBody);
|
||||
initialData.mouth = mapRect(baseAnalysis.mouth, mainBody);
|
||||
initialData.skinColor = baseAnalysis.skinColor;
|
||||
}
|
||||
|
||||
if (blinkAnalysis && stitchBlinkRect) {
|
||||
const be = blinkAnalysis;
|
||||
const minX = Math.min(be.leftEye.x, be.rightEye.x);
|
||||
const minY = Math.min(be.leftEye.y, be.rightEye.y);
|
||||
const maxX = Math.max(be.leftEye.x + be.leftEye.w, be.rightEye.x + be.rightEye.w);
|
||||
const maxY = Math.max(be.leftEye.y + be.leftEye.h, be.rightEye.y + be.rightEye.h);
|
||||
|
||||
const eyesRect = { x: minX, y: minY, w: maxX - minX, h: maxY - minY };
|
||||
initialData.textureClosedEye = mapRect(eyesRect, stitchBlinkRect);
|
||||
}
|
||||
|
||||
if (talkAnalysis && stitchTalkRect) {
|
||||
initialData.textureOpenMouth = mapRect(talkAnalysis.mouth, stitchTalkRect);
|
||||
}
|
||||
|
||||
onAvatarGenerated(imageUrl, name, initialData);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
setError("Failed to process uploaded images. Please ensure they are valid image files.");
|
||||
} finally {
|
||||
setStatus('idle');
|
||||
}
|
||||
};
|
||||
|
||||
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>, setter: (f: File | null) => void) => {
|
||||
if (e.target.files && e.target.files[0]) {
|
||||
setter(e.target.files[0]);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl mx-auto bg-slate-800/50 backdrop-blur-lg border border-slate-700 rounded-2xl shadow-2xl overflow-hidden">
|
||||
<div className="flex border-b border-slate-700">
|
||||
<button
|
||||
onClick={() => setMode('generate')}
|
||||
className={`flex-1 py-4 text-sm font-bold uppercase tracking-wider transition-colors ${
|
||||
mode === 'generate'
|
||||
? 'bg-slate-700/50 text-cyan-400 border-b-2 border-cyan-400'
|
||||
: 'text-slate-500 hover:text-slate-300'
|
||||
}`}
|
||||
>
|
||||
AI Generator
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setMode('upload')}
|
||||
className={`flex-1 py-4 text-sm font-bold uppercase tracking-wider transition-colors ${
|
||||
mode === 'upload'
|
||||
? 'bg-slate-700/50 text-purple-400 border-b-2 border-purple-400'
|
||||
: 'text-slate-500 hover:text-slate-300'
|
||||
}`}
|
||||
>
|
||||
Upload Assets
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="p-8">
|
||||
<div className="text-center mb-8">
|
||||
<h2 className="text-3xl font-bold text-transparent bg-clip-text bg-gradient-to-r from-cyan-400 to-purple-500 mb-2">
|
||||
{mode === 'generate' ? 'Design Your Avatar' : 'Import Your Model'}
|
||||
</h2>
|
||||
<p className="text-slate-400">
|
||||
{mode === 'generate'
|
||||
? 'Describe your dream VTuber model. Gemini will generate a character sheet with expression assets.'
|
||||
: 'Upload your existing character art. We support separate files for blink and talk variants.'
|
||||
}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="space-y-6">
|
||||
<div>
|
||||
<label className="block text-sm font-medium text-slate-300 mb-2">Model Name</label>
|
||||
<input
|
||||
type="text"
|
||||
value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
placeholder="e.g., Neon Kitsune"
|
||||
className="w-full bg-slate-900/50 border border-slate-600 rounded-xl px-4 py-3 text-white placeholder-slate-500 focus:ring-2 focus:ring-cyan-500 focus:border-transparent transition-all outline-none"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{mode === 'generate' ? (
|
||||
<div>
|
||||
<label className="block text-sm font-medium text-slate-300 mb-2">Description</label>
|
||||
<textarea
|
||||
value={prompt}
|
||||
onChange={(e) => setPrompt(e.target.value)}
|
||||
placeholder="e.g., A cyberpunk anime girl with neon blue hair, glowing headphones, wearing a futuristic jacket..."
|
||||
className="w-full h-32 bg-slate-900/50 border border-slate-600 rounded-xl px-4 py-3 text-white placeholder-slate-500 focus:ring-2 focus:ring-cyan-500 focus:border-transparent transition-all outline-none resize-none"
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-4">
|
||||
<div className="p-4 bg-slate-900/50 rounded-xl border border-slate-600 border-dashed">
|
||||
<label className="block text-sm font-bold text-slate-300 mb-2">Base Model (Required)</label>
|
||||
<input type="file" accept="image/*" onChange={(e) => handleFileChange(e, setBaseFile)} className="text-sm text-slate-400 file:mr-4 file:py-2 file:px-4 file:rounded-full file:border-0 file:text-sm file:font-semibold file:bg-cyan-500/10 file:text-cyan-400 hover:file:bg-cyan-500/20"/>
|
||||
<p className="text-xs text-slate-500 mt-1">The main look of your character (Eyes Open, Mouth Closed).</p>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div className="p-4 bg-slate-900/50 rounded-xl border border-slate-600 border-dashed">
|
||||
<label className="block text-sm font-bold text-slate-300 mb-2">Closed Eyes (Optional)</label>
|
||||
<input type="file" accept="image/*" onChange={(e) => handleFileChange(e, setBlinkFile)} className="text-sm text-slate-400 file:mr-4 file:py-2 file:px-4 file:rounded-full file:border-0 file:text-sm file:font-semibold file:bg-purple-500/10 file:text-purple-400 hover:file:bg-purple-500/20"/>
|
||||
</div>
|
||||
<div className="p-4 bg-slate-900/50 rounded-xl border border-slate-600 border-dashed">
|
||||
<label className="block text-sm font-bold text-slate-300 mb-2">Open Mouth (Optional)</label>
|
||||
<input type="file" accept="image/*" onChange={(e) => handleFileChange(e, setTalkFile)} className="text-sm text-slate-400 file:mr-4 file:py-2 file:px-4 file:rounded-full file:border-0 file:text-sm file:font-semibold file:bg-pink-500/10 file:text-pink-400 hover:file:bg-pink-500/20"/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<div className="p-3 bg-red-500/20 border border-red-500/50 rounded-lg text-red-200 text-sm">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<button
|
||||
onClick={mode === 'generate' ? handleGenerate : handleUpload}
|
||||
disabled={status !== 'idle' || !name || (mode === 'generate' && !prompt) || (mode === 'upload' && !baseFile)}
|
||||
className={`w-full py-4 rounded-xl font-bold text-lg transition-all duration-200 ${
|
||||
status !== 'idle' || !name || (mode === 'generate' && !prompt) || (mode === 'upload' && !baseFile)
|
||||
? 'bg-slate-700 text-slate-500 cursor-not-allowed'
|
||||
: 'bg-gradient-to-r from-cyan-500 to-blue-600 hover:from-cyan-400 hover:to-blue-500 text-white shadow-lg shadow-cyan-500/25 transform hover:scale-[1.02]'
|
||||
}`}
|
||||
>
|
||||
{status !== 'idle' ? (
|
||||
<div className="flex items-center justify-center gap-3">
|
||||
<LoadingSpinner />
|
||||
<span>
|
||||
{status === 'generating' ? 'Dreaming up Sheet...' :
|
||||
status === 'stitching' ? 'Processing Assets...' :
|
||||
'Analyzing Features...'}
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
<span>{mode === 'generate' ? 'Generate Model' : 'Create Model'}</span>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" className="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
|
||||
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-8.707l-3-3a1 1 0 00-1.414 1.414L10.586 9H7a1 1 0 100 2h3.586l-1.293 1.293a1 1 0 101.414 1.414l3-3a1 1 0 000-1.414z" clipRule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default AvatarCreator;
|
||||
@@ -0,0 +1,16 @@
|
||||
import React from 'react';
|
||||
|
||||
const ErrorBoundaryFallback: React.FC<{error?: Error | null}> = ({ error }) => (
|
||||
<div style={{ padding: 24, color: 'white' }}>
|
||||
<h2>Something went wrong</h2>
|
||||
<pre style={{ whiteSpace: 'pre-wrap' }}>{String(error)}</pre>
|
||||
</div>
|
||||
);
|
||||
|
||||
// Lightweight functional error boundary placeholder. Replace with a full class-based boundary
|
||||
// if you need to catch render-time errors.
|
||||
const ErrorBoundary: React.FC<{children: React.ReactNode}> = ({ children }) => {
|
||||
return <>{children}</>;
|
||||
};
|
||||
|
||||
export default ErrorBoundary;
|
||||
@@ -0,0 +1,11 @@
|
||||
import React from 'react';
|
||||
|
||||
const LoadingSpinner: React.FC = () => (
|
||||
<div className="flex justify-center items-center space-x-2">
|
||||
<div className="w-4 h-4 bg-cyan-500 rounded-full animate-bounce" style={{ animationDelay: '0s' }}></div>
|
||||
<div className="w-4 h-4 bg-purple-500 rounded-full animate-bounce" style={{ animationDelay: '0.1s' }}></div>
|
||||
<div className="w-4 h-4 bg-pink-500 rounded-full animate-bounce" style={{ animationDelay: '0.2s' }}></div>
|
||||
</div>
|
||||
);
|
||||
|
||||
export default LoadingSpinner;
|
||||
@@ -0,0 +1,269 @@
|
||||
import React, { useState, useRef, useEffect } from 'react';
|
||||
import { Rect } from '../../shared/types';
|
||||
|
||||
interface RiggingEditorProps {
|
||||
imageUrl: string;
|
||||
initialData?: { leftEye: Rect; rightEye: Rect; mouth: Rect; skinColor: string };
|
||||
onComplete: (data: {
|
||||
leftEye: Rect; rightEye: Rect; mouth: Rect; skinColor: string;
|
||||
textureClosedEye: Rect; textureOpenMouth: Rect;
|
||||
mainBody: Rect; chromaKeyColor: string;
|
||||
}) => void;
|
||||
}
|
||||
|
||||
type ActiveFeature = 'leftEye' | 'rightEye' | 'mouth' | 'textureClosedEye' | 'textureOpenMouth' | 'mainBody' | null;
|
||||
|
||||
const ResizableBox: React.FC<{
|
||||
rect: Rect;
|
||||
color: string;
|
||||
label: string;
|
||||
isActive: boolean;
|
||||
onUpdate: (rect: Rect) => void;
|
||||
onActivate: () => void;
|
||||
}> = ({ rect, color, label, isActive, onUpdate, onActivate }) => {
|
||||
const boxRef = useRef<HTMLDivElement>(null);
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const [isResizing, setIsResizing] = useState(false);
|
||||
const startPos = useRef({ x: 0, y: 0 });
|
||||
const startRect = useRef<Rect>({ x: 0, y: 0, w: 0, h: 0 });
|
||||
|
||||
const handleMouseDown = (e: React.MouseEvent) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
onActivate();
|
||||
setIsDragging(true);
|
||||
startPos.current = { x: e.clientX, y: e.clientY };
|
||||
startRect.current = { ...rect };
|
||||
};
|
||||
|
||||
const handleResizeDown = (e: React.MouseEvent) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
onActivate();
|
||||
setIsResizing(true);
|
||||
startPos.current = { x: e.clientX, y: e.clientY };
|
||||
startRect.current = { ...rect };
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const handleMouseMove = (e: MouseEvent) => {
|
||||
if (!isDragging && !isResizing) return;
|
||||
|
||||
const parent = boxRef.current?.parentElement;
|
||||
if (!parent) return;
|
||||
const parentRect = parent.getBoundingClientRect();
|
||||
|
||||
const deltaX = (e.clientX - startPos.current.x) / parentRect.width;
|
||||
const deltaY = (e.clientY - startPos.current.y) / parentRect.height;
|
||||
|
||||
if (isDragging) {
|
||||
onUpdate({
|
||||
...rect,
|
||||
x: startRect.current.x + deltaX,
|
||||
y: startRect.current.y + deltaY,
|
||||
});
|
||||
} else if (isResizing) {
|
||||
onUpdate({
|
||||
...rect,
|
||||
w: Math.max(0.01, startRect.current.w + deltaX),
|
||||
h: Math.max(0.01, startRect.current.h + deltaY),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleMouseUp = () => {
|
||||
setIsDragging(false);
|
||||
setIsResizing(false);
|
||||
};
|
||||
|
||||
if (isDragging || isResizing) {
|
||||
window.addEventListener('mousemove', handleMouseMove);
|
||||
window.addEventListener('mouseup', handleMouseUp);
|
||||
}
|
||||
|
||||
return () => {
|
||||
window.removeEventListener('mousemove', handleMouseMove);
|
||||
window.removeEventListener('mouseup', handleMouseUp);
|
||||
};
|
||||
}, [isDragging, isResizing, rect, onUpdate]);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={boxRef}
|
||||
onMouseDown={handleMouseDown}
|
||||
className={`absolute border-2 cursor-move group transition-colors ${isActive ? 'z-30' : 'z-20'}`}
|
||||
style={{
|
||||
left: `${rect.x * 100}%`,
|
||||
top: `${rect.y * 100}%`,
|
||||
width: `${rect.w * 100}%`,
|
||||
height: `${rect.h * 100}%`,
|
||||
borderColor: color,
|
||||
backgroundColor: isActive ? `${color}20` : 'transparent',
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className="absolute -top-6 left-0 text-xs font-bold px-1 rounded text-white whitespace-nowrap shadow-sm"
|
||||
style={{ backgroundColor: color }}
|
||||
>
|
||||
{label}
|
||||
</div>
|
||||
|
||||
<div
|
||||
onMouseDown={handleResizeDown}
|
||||
className="absolute bottom-0 right-0 w-4 h-4 bg-white border-2 cursor-nwse-resize opacity-0 group-hover:opacity-100 transition-opacity"
|
||||
style={{ borderColor: color }}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, onComplete }) => {
|
||||
const [leftEye, setLeftEye] = useState<Rect>(initialData?.leftEye || { x: 0.25, y: 0.4, w: 0.1, h: 0.1 });
|
||||
const [rightEye, setRightEye] = useState<Rect>(initialData?.rightEye || { x: 0.45, y: 0.4, w: 0.1, h: 0.1 });
|
||||
const [mouth, setMouth] = useState<Rect>(initialData?.mouth || { x: 0.35, y: 0.55, w: 0.1, h: 0.05 });
|
||||
|
||||
const [mainBody, setMainBody] = useState<Rect>({ x: 0.05, y: 0.05, w: 0.65, h: 0.9 });
|
||||
|
||||
const [textureClosedEye, setTextureClosedEye] = useState<Rect>({ x: 0.7, y: 0.1, w: 0.2, h: 0.2 });
|
||||
const [textureOpenMouth, setTextureOpenMouth] = useState<Rect>({ x: 0.7, y: 0.5, w: 0.2, h: 0.2 });
|
||||
|
||||
const [skinColor, setSkinColor] = useState<string>(initialData?.skinColor || '#fcd3bf');
|
||||
const [useAiBackground, setUseAiBackground] = useState<boolean>(true);
|
||||
|
||||
const [activeFeature, setActiveFeature] = useState<ActiveFeature>(null);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col items-center h-full max-w-6xl mx-auto p-4">
|
||||
<div className="text-center mb-6">
|
||||
<h2 className="text-2xl font-bold text-white mb-2">Rig Your Character</h2>
|
||||
<p className="text-slate-400 text-sm">
|
||||
1. Adjust the <b>Main Body</b> (Yellow) to frame your character.<br/>
|
||||
2. Match the <b>Targets</b> (Red/Blue/Green) to the face features.<br/>
|
||||
3. Match the <b>Sources</b> (Purple/Orange) to the assets on the right.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="flex gap-6 w-full items-start h-[70vh]">
|
||||
<div className="flex-1 bg-slate-800 p-4 rounded-xl border border-slate-700 flex justify-center h-full overflow-hidden relative">
|
||||
<div className="relative inline-block h-full">
|
||||
<img
|
||||
src={imageUrl}
|
||||
alt="Rigging Target"
|
||||
className="h-full w-auto object-contain rounded-lg pointer-events-none select-none block"
|
||||
draggable={false}
|
||||
/>
|
||||
|
||||
<div className="absolute inset-0 w-full h-full">
|
||||
<ResizableBox
|
||||
rect={mainBody} color="#facc15" label="Main Body"
|
||||
isActive={activeFeature === 'mainBody'}
|
||||
onUpdate={setMainBody} onActivate={() => setActiveFeature('mainBody')}
|
||||
/>
|
||||
|
||||
<ResizableBox
|
||||
rect={leftEye} color="#ef4444" label="Left Eye Target"
|
||||
isActive={activeFeature === 'leftEye'}
|
||||
onUpdate={setLeftEye} onActivate={() => setActiveFeature('leftEye')}
|
||||
/>
|
||||
<ResizableBox
|
||||
rect={rightEye} color="#3b82f6" label="Right Eye Target"
|
||||
isActive={activeFeature === 'rightEye'}
|
||||
onUpdate={setRightEye} onActivate={() => setActiveFeature('rightEye')}
|
||||
/>
|
||||
<ResizableBox
|
||||
rect={mouth} color="#22c55e" label="Mouth Target"
|
||||
isActive={activeFeature === 'mouth'}
|
||||
onUpdate={setMouth} onActivate={() => setActiveFeature('mouth')}
|
||||
/>
|
||||
|
||||
<ResizableBox
|
||||
rect={textureClosedEye} color="#a855f7" label="Source: Closed Eyes"
|
||||
isActive={activeFeature === 'textureClosedEye'}
|
||||
onUpdate={setTextureClosedEye} onActivate={() => setActiveFeature('textureClosedEye')}
|
||||
/>
|
||||
<ResizableBox
|
||||
rect={textureOpenMouth} color="#f97316" label="Source: Open Mouth"
|
||||
isActive={activeFeature === 'textureOpenMouth'}
|
||||
onUpdate={setTextureOpenMouth} onActivate={() => setActiveFeature('textureOpenMouth')}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="w-72 flex flex-col gap-4 bg-slate-800/50 p-6 rounded-xl border border-slate-700 h-full overflow-y-auto">
|
||||
|
||||
<div className="bg-slate-900/50 p-4 rounded-lg space-y-3">
|
||||
<div>
|
||||
<label className="block text-xs font-bold text-slate-400 mb-2 uppercase">Background Removal</label>
|
||||
<div className="flex items-center justify-between p-2 bg-slate-800 rounded-lg border border-slate-700">
|
||||
<span className="text-xs text-slate-300">AI Magic Removal</span>
|
||||
<label className="relative inline-flex items-center cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
className="sr-only peer"
|
||||
checked={useAiBackground}
|
||||
onChange={(e) => setUseAiBackground(e.target.checked)}
|
||||
/>
|
||||
<div className="w-9 h-5 bg-slate-600 peer-focus:outline-none rounded-full peer peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-[2px] after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-4 after:w-4 after:transition-all peer-checked:bg-cyan-500"></div>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<label className="block text-xs font-bold text-slate-400 mb-1 uppercase">Eyelid Skin Color</label>
|
||||
<div className="flex items-center gap-3">
|
||||
<input
|
||||
type="color"
|
||||
value={skinColor}
|
||||
onChange={(e) => setSkinColor(e.target.value)}
|
||||
className="w-8 h-8 rounded cursor-pointer border-0 p-0"
|
||||
/>
|
||||
<span className="text-xs text-slate-400 font-mono">Fallback</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-3 flex-1">
|
||||
<div className="text-xs font-bold text-slate-400 uppercase border-b border-slate-700 pb-1">Composition</div>
|
||||
<div className="flex items-center gap-2 text-sm text-slate-300 cursor-pointer hover:text-white" onClick={() => setActiveFeature('mainBody')}>
|
||||
<div className="w-3 h-3 bg-yellow-400 rounded-full shadow"></div> Main Body Crop
|
||||
</div>
|
||||
|
||||
<div className="text-xs font-bold text-slate-400 uppercase border-b border-slate-700 pb-1 mt-4">Targets (Main Face)</div>
|
||||
<div className="flex items-center gap-2 text-sm text-slate-300 cursor-pointer hover:text-white" onClick={() => setActiveFeature('leftEye')}>
|
||||
<div className="w-3 h-3 bg-red-500 rounded-full shadow"></div> Left Eye
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-sm text-slate-300 cursor-pointer hover:text-white" onClick={() => setActiveFeature('rightEye')}>
|
||||
<div className="w-3 h-3 bg-blue-500 rounded-full shadow"></div> Right Eye
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-sm text-slate-300 cursor-pointer hover:text-white" onClick={() => setActiveFeature('mouth')}>
|
||||
<div className="w-3 h-3 bg-green-500 rounded-full shadow"></div> Mouth
|
||||
</div>
|
||||
|
||||
<div className="text-xs font-bold text-slate-400 uppercase border-b border-slate-700 pb-1 mt-4">Sources (Right Side)</div>
|
||||
<div className="flex items-center gap-2 text-sm text-slate-300 cursor-pointer hover:text-white" onClick={() => setActiveFeature('textureClosedEye')}>
|
||||
<div className="w-3 h-3 bg-purple-500 rounded-full shadow"></div> Closed Eye Texture
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-sm text-slate-300 cursor-pointer hover:text-white" onClick={() => setActiveFeature('textureOpenMouth')}>
|
||||
<div className="w-3 h-3 bg-orange-500 rounded-full shadow"></div> Open Mouth Texture
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-4">
|
||||
<button
|
||||
onClick={() => onComplete({
|
||||
leftEye, rightEye, mouth, skinColor,
|
||||
textureClosedEye, textureOpenMouth, mainBody,
|
||||
chromaKeyColor: useAiBackground ? 'AI_AUTO' : ''
|
||||
})}
|
||||
className="w-full py-4 bg-gradient-to-r from-cyan-500 to-blue-600 hover:from-cyan-400 hover:to-blue-500 text-white rounded-xl font-bold shadow-lg shadow-cyan-500/25 transform hover:scale-[1.02] transition-all"
|
||||
>
|
||||
Finish Rigging
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default RiggingEditor;
|
||||
@@ -0,0 +1,299 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import { useFaceTracking } from '../hooks/useFaceTracking';
|
||||
import { removeBackground } from '../services/visionService';
|
||||
import { AvatarConfig, Rect } from '../../shared/types';
|
||||
import LoadingSpinner from './LoadingSpinner';
|
||||
|
||||
interface StudioProps {
|
||||
avatar: AvatarConfig;
|
||||
onBack: () => void;
|
||||
}
|
||||
|
||||
const Sprite: React.FC<{
|
||||
imageSrc: string;
|
||||
sourceRect: Rect;
|
||||
style?: React.CSSProperties;
|
||||
className?: string;
|
||||
}> = ({ imageSrc, sourceRect, style, className }) => {
|
||||
const widthScale = 100 / (sourceRect.w * 100);
|
||||
const heightScale = 100 / (sourceRect.h * 100);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`overflow-hidden relative ${className}`}
|
||||
style={style}
|
||||
>
|
||||
<img
|
||||
src={imageSrc}
|
||||
alt=""
|
||||
style={{
|
||||
position: 'absolute',
|
||||
top: `-${sourceRect.y * 100 * heightScale}%`,
|
||||
left: `-${sourceRect.x * 100 * widthScale}%`,
|
||||
width: `${widthScale * 100}%`,
|
||||
height: `${heightScale * 100}%`,
|
||||
maxWidth: 'none',
|
||||
maxHeight: 'none',
|
||||
pointerEvents: 'none'
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const [cameraReady, setCameraReady] = useState(false);
|
||||
const [processedImageUrl, setProcessedImageUrl] = useState<string | null>(null);
|
||||
|
||||
const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current);
|
||||
|
||||
useEffect(() => {
|
||||
const startCamera = async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { width: 640, height: 480 },
|
||||
audio: false
|
||||
});
|
||||
if (videoRef.current) {
|
||||
videoRef.current.srcObject = stream;
|
||||
videoRef.current.onloadeddata = () => {
|
||||
setCameraReady(true);
|
||||
if ((window as any).electronLog) (window as any).electronLog.info('Camera ready');
|
||||
};
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error accessing camera:", err);
|
||||
alert("Could not access camera. Please ensure permissions are granted.");
|
||||
if ((window as any).electronLog) (window as any).electronLog.error(`Camera access failed: ${String(err)}`);
|
||||
}
|
||||
};
|
||||
|
||||
startCamera();
|
||||
|
||||
return () => {
|
||||
if (videoRef.current && videoRef.current.srcObject) {
|
||||
const stream = videoRef.current.srcObject as MediaStream;
|
||||
stream.getTracks().forEach(track => track.stop());
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!avatar.chromaKeyColor) {
|
||||
setProcessedImageUrl(avatar.imageUrl);
|
||||
return;
|
||||
}
|
||||
|
||||
const process = async () => {
|
||||
const result = await removeBackground(avatar.imageUrl);
|
||||
setProcessedImageUrl(result);
|
||||
};
|
||||
|
||||
process();
|
||||
}, [avatar.imageUrl, avatar.chromaKeyColor]);
|
||||
|
||||
useEffect(() => {
|
||||
if (cameraReady && !isModelLoading) {
|
||||
startTracking();
|
||||
}
|
||||
}, [cameraReady, isModelLoading, startTracking]);
|
||||
|
||||
const getAvatarStyle = () => {
|
||||
const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val;
|
||||
|
||||
const rX = smooth(trackingData.rotationX);
|
||||
const rY = smooth(trackingData.rotationY);
|
||||
const rZ = smooth(trackingData.rotationZ);
|
||||
const tX = smooth(trackingData.translationX);
|
||||
const tY = smooth(trackingData.translationY);
|
||||
|
||||
const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0;
|
||||
|
||||
return {
|
||||
transform: `
|
||||
translate(${tX * 150}px, ${tY * 100 + bounce}px)
|
||||
rotate(${rZ * 1}rad)
|
||||
perspective(500px)
|
||||
rotateX(${rX * 15}deg)
|
||||
rotateY(${rY * -25}deg)
|
||||
scale(${1 + trackingData.mouthOpen * 0.02})
|
||||
`,
|
||||
filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`,
|
||||
transition: 'transform 0.1s ease-out, filter 0.1s ease'
|
||||
} as React.CSSProperties;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="h-screen w-full flex flex-col bg-slate-900 overflow-hidden relative">
|
||||
<video
|
||||
ref={videoRef}
|
||||
autoPlay
|
||||
playsInline
|
||||
muted
|
||||
className="absolute opacity-0 pointer-events-none w-1 h-1"
|
||||
/>
|
||||
|
||||
<div className="absolute top-0 left-0 right-0 z-20 p-4 flex justify-between items-center bg-gradient-to-b from-slate-900 to-transparent">
|
||||
<button
|
||||
onClick={onBack}
|
||||
className="px-4 py-2 bg-slate-800/80 hover:bg-slate-700 backdrop-blur rounded-lg text-white font-medium transition-colors border border-slate-600"
|
||||
>
|
||||
← Exit Studio
|
||||
</button>
|
||||
<div className="flex gap-2">
|
||||
<div className={`px-3 py-1 rounded-full text-xs font-bold flex items-center gap-2 ${isModelLoading ? 'bg-yellow-500/20 text-yellow-400' : 'bg-green-500/20 text-green-400'}`}>
|
||||
<span className={`w-2 h-2 rounded-full ${isModelLoading ? 'bg-yellow-400 animate-pulse' : 'bg-green-400'}`}></span>
|
||||
{isModelLoading ? 'Loading Vision Model...' : 'Tracking Active'}
|
||||
</div>
|
||||
<div className="px-3 py-1 rounded-full text-xs font-bold bg-purple-500/20 text-purple-400 border border-purple-500/30">
|
||||
{avatar.name}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex-1 relative flex items-center justify-center overflow-hidden">
|
||||
<div className="absolute inset-0 opacity-20"
|
||||
style={{
|
||||
backgroundImage: 'radial-gradient(#4f46e5 1px, transparent 1px)',
|
||||
backgroundSize: '30px 30px'
|
||||
}}>
|
||||
</div>
|
||||
|
||||
<div className="absolute inset-0 bg-gradient-to-t from-slate-900 via-transparent to-slate-900 pointer-events-none"></div>
|
||||
|
||||
<div className="relative w-[600px] h-[600px] flex items-center justify-center z-10">
|
||||
{!processedImageUrl ? (
|
||||
<div className="flex flex-col items-center justify-center gap-4">
|
||||
<LoadingSpinner />
|
||||
<span className="text-cyan-400 font-mono text-sm">REMOVING BACKGROUND...</span>
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
className="relative w-full h-full flex items-center justify-center"
|
||||
style={getAvatarStyle()}
|
||||
>
|
||||
{avatar.mainBody ? (
|
||||
<Sprite
|
||||
imageSrc={processedImageUrl}
|
||||
sourceRect={avatar.mainBody}
|
||||
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
|
||||
/>
|
||||
) : (
|
||||
<img
|
||||
src={processedImageUrl}
|
||||
alt="Avatar"
|
||||
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
|
||||
/>
|
||||
)}
|
||||
|
||||
{avatar.leftEye && avatar.textureClosedEye && (
|
||||
<Sprite
|
||||
imageSrc={processedImageUrl}
|
||||
sourceRect={avatar.textureClosedEye}
|
||||
className="absolute pointer-events-none z-20"
|
||||
style={{
|
||||
left: `${avatar.leftEye.x * 100}%`,
|
||||
top: `${avatar.leftEye.y * 100}%`,
|
||||
width: `${avatar.leftEye.w * 100}%`,
|
||||
height: `${avatar.leftEye.h * 100}%`,
|
||||
opacity: trackingData.isBlinkingLeft ? 1 : 0,
|
||||
transition: 'opacity 0.05s linear',
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{avatar.rightEye && avatar.textureClosedEye && (
|
||||
<Sprite
|
||||
imageSrc={processedImageUrl}
|
||||
sourceRect={avatar.textureClosedEye}
|
||||
className="absolute pointer-events-none z-20"
|
||||
style={{
|
||||
left: `${avatar.rightEye.x * 100}%`,
|
||||
top: `${avatar.rightEye.y * 100}%`,
|
||||
width: `${avatar.rightEye.w * 100}%`,
|
||||
height: `${avatar.rightEye.h * 100}%`,
|
||||
opacity: trackingData.isBlinkingRight ? 1 : 0,
|
||||
transition: 'opacity 0.05s linear',
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
|
||||
{avatar.mouth && avatar.textureOpenMouth && (
|
||||
<div
|
||||
className="absolute pointer-events-none flex items-center justify-center z-10"
|
||||
style={{
|
||||
left: `${avatar.mouth.x * 100}%`,
|
||||
top: `${avatar.mouth.y * 100}%`,
|
||||
width: `${avatar.mouth.w * 100}%`,
|
||||
height: `${avatar.mouth.h * 100}%`,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className="absolute w-[120%] h-[120%] transition-opacity duration-75"
|
||||
style={{
|
||||
backgroundColor: avatar.skinColor || '#fcd3bf',
|
||||
opacity: trackingData.mouthOpen > 0.1 ? 1 : 0,
|
||||
filter: 'blur(4px)',
|
||||
borderRadius: '50%'
|
||||
}}
|
||||
/>
|
||||
|
||||
<Sprite
|
||||
imageSrc={processedImageUrl}
|
||||
sourceRect={avatar.textureOpenMouth}
|
||||
className="w-full h-full"
|
||||
style={{
|
||||
opacity: trackingData.mouthOpen > 0.05 ? 1 : 0,
|
||||
transform: `scaleY(${0.8 + trackingData.mouthOpen * 0.5})`,
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(!cameraReady) && (
|
||||
<div className="absolute inset-0 flex items-center justify-center bg-slate-900/80 z-20 rounded-xl backdrop-blur-sm">
|
||||
<div className="text-cyan-400 animate-pulse font-mono">INITIALIZING CAMERA LINK...</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="h-24 bg-slate-800 border-t border-slate-700 p-4 flex justify-center items-center gap-6 z-20">
|
||||
<div className="flex flex-col items-center">
|
||||
<span className="text-xs text-slate-400 mb-1 font-mono">MOUTH</span>
|
||||
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden">
|
||||
<div className="h-full bg-cyan-400 transition-all duration-75" style={{ width: `${Math.min(trackingData.mouthOpen * 100, 100)}%` }}></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col items-center">
|
||||
<span className="text-xs text-slate-400 mb-1 font-mono">HEAD ROLL</span>
|
||||
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden flex justify-center relative">
|
||||
<div className="absolute w-[1px] h-full bg-slate-500 left-1/2"></div>
|
||||
<div
|
||||
className="h-full bg-purple-500 transition-all duration-75 absolute"
|
||||
style={{
|
||||
width: `${Math.abs(trackingData.rotationZ * 50)}%`,
|
||||
left: trackingData.rotationZ < 0 ? 'auto' : '50%',
|
||||
right: trackingData.rotationZ < 0 ? '50%' : 'auto'
|
||||
}}
|
||||
></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col items-center">
|
||||
<span className="text-xs text-slate-400 mb-1 font-mono">BLINK</span>
|
||||
<div className="flex gap-2">
|
||||
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingLeft ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
||||
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingRight ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default Studio;
|
||||
@@ -0,0 +1,130 @@
|
||||
import { useEffect, useRef, useState, useCallback } from 'react';
|
||||
import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
|
||||
import { TrackingData } from '../../shared/types';
|
||||
|
||||
export const useFaceTracking = (videoElement: HTMLVideoElement | null) => {
|
||||
const [isTracking, setIsTracking] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const faceLandmarkerRef = useRef<FaceLandmarker | null>(null);
|
||||
const requestRef = useRef<number | null>(null);
|
||||
const lastVideoTimeRef = useRef<number>(-1);
|
||||
const [trackingData, setTrackingData] = useState<TrackingData>({
|
||||
rotationX: 0,
|
||||
rotationY: 0,
|
||||
rotationZ: 0,
|
||||
translationX: 0,
|
||||
translationY: 0,
|
||||
mouthOpen: 0,
|
||||
isBlinkingLeft: false,
|
||||
isBlinkingRight: false,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
const initMediaPipe = async () => {
|
||||
try {
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(
|
||||
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm"
|
||||
);
|
||||
|
||||
faceLandmarkerRef.current = await FaceLandmarker.createFromOptions(filesetResolver, {
|
||||
baseOptions: {
|
||||
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
|
||||
delegate: "GPU"
|
||||
},
|
||||
outputFaceBlendshapes: true,
|
||||
outputFacialTransformationMatrixes: true,
|
||||
runningMode: "VIDEO",
|
||||
numFaces: 1
|
||||
});
|
||||
|
||||
setIsLoading(false);
|
||||
if ((window as any).electronLog) (window as any).electronLog.info('MediaPipe faceLandmarker loaded');
|
||||
} catch (error) {
|
||||
console.error("Failed to load MediaPipe:", error);
|
||||
if ((window as any).electronLog) (window as any).electronLog.error(`Failed to load MediaPipe: ${String(error)}`);
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
initMediaPipe();
|
||||
|
||||
return () => {
|
||||
faceLandmarkerRef.current?.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
const predict = useCallback(() => {
|
||||
if (!faceLandmarkerRef.current || !videoElement) return;
|
||||
|
||||
if (videoElement.readyState < 2) return;
|
||||
|
||||
const nowInMs = Date.now();
|
||||
if (lastVideoTimeRef.current !== videoElement.currentTime) {
|
||||
lastVideoTimeRef.current = videoElement.currentTime;
|
||||
|
||||
const results = faceLandmarkerRef.current.detectForVideo(videoElement, nowInMs);
|
||||
|
||||
if (results.faceLandmarks && results.faceLandmarks.length > 0) {
|
||||
const blendshapes = results.faceBlendshapes?.[0]?.categories;
|
||||
|
||||
let mouthOpen = 0;
|
||||
let eyeBlinkLeft = 0;
|
||||
let eyeBlinkRight = 0;
|
||||
|
||||
if (blendshapes) {
|
||||
mouthOpen = blendshapes.find(c => c.categoryName === 'jawOpen')?.score || 0;
|
||||
eyeBlinkLeft = blendshapes.find(c => c.categoryName === 'eyeBlinkLeft')?.score || 0;
|
||||
eyeBlinkRight = blendshapes.find(c => c.categoryName === 'eyeBlinkRight')?.score || 0;
|
||||
}
|
||||
|
||||
const landmarks = results.faceLandmarks[0];
|
||||
const leftEye = landmarks[33];
|
||||
const rightEye = landmarks[263];
|
||||
const dy = rightEye.y - leftEye.y;
|
||||
const dx = rightEye.x - leftEye.x;
|
||||
const roll = Math.atan2(dy, dx);
|
||||
|
||||
const nose = landmarks[1];
|
||||
const midPointX = (leftEye.x + rightEye.x) / 2;
|
||||
const yaw = (nose.x - midPointX) * 2;
|
||||
|
||||
const midPointY = (leftEye.y + rightEye.y) / 2;
|
||||
const pitch = (nose.y - midPointY) * 2;
|
||||
|
||||
const transX = (nose.x - 0.5) * 2;
|
||||
const transY = (nose.y - 0.5) * 2;
|
||||
|
||||
setTrackingData({
|
||||
rotationZ: roll,
|
||||
rotationY: yaw,
|
||||
rotationX: pitch,
|
||||
translationX: transX,
|
||||
translationY: transY,
|
||||
mouthOpen,
|
||||
isBlinkingLeft: eyeBlinkLeft > 0.5,
|
||||
isBlinkingRight: eyeBlinkRight > 0.5
|
||||
});
|
||||
}
|
||||
}
|
||||
requestRef.current = requestAnimationFrame(predict);
|
||||
}, [videoElement]);
|
||||
|
||||
const startTracking = useCallback(() => {
|
||||
setIsTracking(true);
|
||||
requestRef.current = requestAnimationFrame(predict);
|
||||
}, [predict]);
|
||||
|
||||
const stopTracking = useCallback(() => {
|
||||
setIsTracking(false);
|
||||
if (requestRef.current) {
|
||||
cancelAnimationFrame(requestRef.current);
|
||||
}
|
||||
}, []);
|
||||
|
||||
return {
|
||||
isLoading,
|
||||
trackingData,
|
||||
startTracking,
|
||||
stopTracking
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
import React from 'react';
|
||||
import ReactDOM from 'react-dom/client';
|
||||
import App from './App';
|
||||
import './styles.css';
|
||||
import ErrorBoundary from './components/ErrorBoundary';
|
||||
|
||||
const rootElement = document.getElementById('root');
|
||||
if (!rootElement) {
|
||||
throw new Error("Could not find root element to mount to");
|
||||
}
|
||||
|
||||
const root = ReactDOM.createRoot(rootElement);
|
||||
if ((window as any).electronLog) {
|
||||
(window as any).electronLog.info('renderer started, mounting React root');
|
||||
}
|
||||
|
||||
root.render(
|
||||
<React.StrictMode>
|
||||
<ErrorBoundary>
|
||||
<App />
|
||||
</ErrorBoundary>
|
||||
</React.StrictMode>
|
||||
);
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { } from '@google/genai';
|
||||
|
||||
const placeholderGenerate = async (prompt: string) => {
|
||||
const text = encodeURIComponent((prompt || '').substring(0, 80));
|
||||
return `data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='1400' height='900'><rect width='100%' height='100%' fill='%230f172a' /><text x='50%' y='50%' fill='white' font-size='40' font-family='Inter' dominant-baseline='middle' text-anchor='middle'>Placeholder: ${text}</text></svg>`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate an avatar image. Flow:
|
||||
* 1) If running in Electron with the preload available, call main via window.electronAPI (passes localStorage key)
|
||||
* 2) Else, attempt to dynamically import @google/genai and call it from the renderer using key in localStorage
|
||||
* 3) Fallback to placeholder
|
||||
*/
|
||||
export const generateAvatarImage = async (description: string): Promise<string> => {
|
||||
try {
|
||||
// Prefer the Electron main process if available (safer, no browser CORS issues)
|
||||
if (typeof window !== 'undefined' && (window as any).electronAPI && (window as any).electronAPI.generateAvatar) {
|
||||
try {
|
||||
const apiKey = localStorage.getItem('GEMINI_API_KEY') || undefined;
|
||||
const res = await (window as any).electronAPI.generateAvatar(description, apiKey);
|
||||
if (res && res.image) return res.image;
|
||||
} catch (e) {
|
||||
console.warn('[geminiService] electronAPI.generateAvatar failed, falling through to client attempt', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Try running the GenAI client in the renderer (may fail due to environment/CORS)
|
||||
try {
|
||||
const apiKey = localStorage.getItem('GEMINI_API_KEY') || (window as any)?.process?.env?.GEMINI_API_KEY || (window as any)?.process?.env?.API_KEY;
|
||||
if (!apiKey) {
|
||||
console.warn('[geminiService] No API key available for client-side generation');
|
||||
return await placeholderGenerate(description);
|
||||
}
|
||||
|
||||
// Dynamic import so bundlers only include this if actually used
|
||||
const mod = await import('@google/genai');
|
||||
const { GoogleGenAI } = mod as any;
|
||||
const ai = new GoogleGenAI({ apiKey });
|
||||
|
||||
const prompt = `
|
||||
Create a VTuber character sheet with a flat 2D anime style.
|
||||
|
||||
LAYOUT:
|
||||
1. MAIN CHARACTER (Left side, takes up 70% of width):
|
||||
- Front-facing view, head and shoulders.
|
||||
- Neutral expression, eyes open, mouth closed.
|
||||
|
||||
2. EXPRESSION ASSETS (Right side, vertical column):
|
||||
- Top: The same character's face with EYES CLOSED (for blinking).
|
||||
- Bottom: The same character's face with MOUTH OPEN (for talking).
|
||||
|
||||
Character Description: ${description}
|
||||
|
||||
Style: Vibrant, clean lines, solid white or green background for easy keying.
|
||||
`;
|
||||
|
||||
const response = await ai.models.generateContent({
|
||||
model: 'gemini-3-pro-image-preview',
|
||||
contents: {
|
||||
parts: [{ text: prompt }]
|
||||
},
|
||||
config: {
|
||||
imageConfig: {
|
||||
aspectRatio: '16:9',
|
||||
imageSize: '1K'
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const parts = response?.candidates?.[0]?.content?.parts || [];
|
||||
for (const part of parts) {
|
||||
if (part.inlineData && part.inlineData.data) {
|
||||
return `data:image/png;base64,${part.inlineData.data}`;
|
||||
}
|
||||
}
|
||||
|
||||
console.warn('[geminiService] No image data found in client response');
|
||||
return await placeholderGenerate(description);
|
||||
} catch (clientErr) {
|
||||
console.warn('[geminiService] Client-side GenAI attempt failed:', clientErr);
|
||||
return await placeholderGenerate(description);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[geminiService] Unexpected error:', err);
|
||||
return await placeholderGenerate(description);
|
||||
}
|
||||
};
|
||||
|
||||
export default { generateAvatarImage };
|
||||
@@ -0,0 +1,87 @@
|
||||
import { Rect } from '../../shared/types';
|
||||
|
||||
export const fileToDataUrl = (file: File): Promise<string> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = (e) => resolve(e.target?.result as string);
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
};
|
||||
|
||||
export const loadImage = (src: string): Promise<HTMLImageElement> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.crossOrigin = "anonymous";
|
||||
img.onload = () => resolve(img);
|
||||
img.onerror = reject;
|
||||
img.src = src;
|
||||
});
|
||||
};
|
||||
|
||||
export const stitchAssets = async (
|
||||
baseSrc: string,
|
||||
blinkSrc?: string,
|
||||
talkSrc?: string
|
||||
): Promise<{ imageUrl: string; mainBody: Rect; textureClosedEye?: Rect; textureOpenMouth?: Rect }> => {
|
||||
const baseImg = await loadImage(baseSrc);
|
||||
const blinkImg = blinkSrc ? await loadImage(blinkSrc) : null;
|
||||
const talkImg = talkSrc ? await loadImage(talkSrc) : null;
|
||||
|
||||
const sidebarWidth = Math.max(blinkImg?.width || 0, talkImg?.width || 0);
|
||||
|
||||
if (sidebarWidth === 0) {
|
||||
return {
|
||||
imageUrl: baseSrc,
|
||||
mainBody: { x: 0, y: 0, w: 1, h: 1 }
|
||||
};
|
||||
}
|
||||
|
||||
const totalWidth = baseImg.width + sidebarWidth;
|
||||
const totalHeight = Math.max(baseImg.height, (blinkImg?.height || 0) + (talkImg?.height || 0));
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = totalWidth;
|
||||
canvas.height = totalHeight;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) throw new Error("Could not get canvas context");
|
||||
|
||||
ctx.drawImage(baseImg, 0, 0);
|
||||
|
||||
const mainBody: Rect = {
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: baseImg.width / totalWidth,
|
||||
h: baseImg.height / totalHeight
|
||||
};
|
||||
|
||||
let textureClosedEye: Rect | undefined;
|
||||
if (blinkImg) {
|
||||
ctx.drawImage(blinkImg, baseImg.width, 0);
|
||||
textureClosedEye = {
|
||||
x: baseImg.width / totalWidth,
|
||||
y: 0,
|
||||
w: blinkImg.width / totalWidth,
|
||||
h: blinkImg.height / totalHeight
|
||||
};
|
||||
}
|
||||
|
||||
let textureOpenMouth: Rect | undefined;
|
||||
if (talkImg) {
|
||||
const yPos = blinkImg ? blinkImg.height : 0;
|
||||
ctx.drawImage(talkImg, baseImg.width, yPos);
|
||||
textureOpenMouth = {
|
||||
x: baseImg.width / totalWidth,
|
||||
y: yPos / totalHeight,
|
||||
w: talkImg.width / totalWidth,
|
||||
h: talkImg.height / totalHeight
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
imageUrl: canvas.toDataURL('image/png'),
|
||||
mainBody,
|
||||
textureClosedEye,
|
||||
textureOpenMouth
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,221 @@
|
||||
import { FaceLandmarker, FilesetResolver, ImageSegmenter } from '@mediapipe/tasks-vision';
|
||||
import { Rect } from '../../shared/types';
|
||||
|
||||
let faceLandmarker: FaceLandmarker | null = null;
|
||||
let imageSegmenter: ImageSegmenter | null = null;
|
||||
|
||||
const initVision = async () => {
|
||||
if (faceLandmarker) return;
|
||||
|
||||
try {
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(
|
||||
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm"
|
||||
);
|
||||
|
||||
// Use CPU delegate in Electron/dev for deterministic behavior. GPU can be flaky across
|
||||
// platforms and in headless contexts; switch to GPU later if you need performance.
|
||||
faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
|
||||
baseOptions: {
|
||||
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
|
||||
delegate: "CPU"
|
||||
},
|
||||
runningMode: "IMAGE",
|
||||
numFaces: 1
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("Failed to initialize vision service:", e);
|
||||
}
|
||||
};
|
||||
|
||||
const initSegmenter = async () => {
|
||||
if (imageSegmenter) return;
|
||||
|
||||
try {
|
||||
const filesetResolver = await FilesetResolver.forVisionTasks(
|
||||
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm"
|
||||
);
|
||||
|
||||
imageSegmenter = await ImageSegmenter.createFromOptions(filesetResolver, {
|
||||
baseOptions: {
|
||||
modelAssetPath: "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite",
|
||||
delegate: "GPU"
|
||||
},
|
||||
runningMode: "IMAGE",
|
||||
outputCategoryMask: false,
|
||||
outputConfidenceMasks: true
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("Failed to initialize segmenter:", e);
|
||||
}
|
||||
};
|
||||
|
||||
export const analyzeAvatarImage = async (imageUrl: string): Promise<{ leftEye: Rect, rightEye: Rect, mouth: Rect, skinColor: string } | null> => {
|
||||
try {
|
||||
await initVision();
|
||||
if (!faceLandmarker) return null;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.crossOrigin = "anonymous";
|
||||
img.onload = () => {
|
||||
try {
|
||||
// If the generated image is small, upscale it to improve detection reliability
|
||||
const minSide = Math.min(img.width, img.height);
|
||||
let detectorInput: HTMLImageElement | HTMLCanvasElement = img;
|
||||
if (minSide < 256) {
|
||||
const scale = Math.ceil(256 / minSide);
|
||||
const c = document.createElement('canvas');
|
||||
c.width = img.width * scale;
|
||||
c.height = img.height * scale;
|
||||
const cctx = c.getContext('2d');
|
||||
if (cctx) cctx.drawImage(img, 0, 0, c.width, c.height);
|
||||
detectorInput = c;
|
||||
console.log('[vision] upscaled image for detection to', c.width, 'x', c.height);
|
||||
} else {
|
||||
console.log('[vision] using image size', img.width, 'x', img.height);
|
||||
}
|
||||
|
||||
const result = faceLandmarker!.detect(detectorInput as any);
|
||||
console.log('[vision] detection result', result);
|
||||
|
||||
if (result.faceLandmarks && result.faceLandmarks.length > 0) {
|
||||
const landmarks = result.faceLandmarks[0];
|
||||
|
||||
const getRect = (indices: number[]): Rect => {
|
||||
let minX = 1, minY = 1, maxX = 0, maxY = 0;
|
||||
|
||||
indices.forEach(i => {
|
||||
const l = landmarks[i];
|
||||
if (l.x < minX) minX = l.x;
|
||||
if (l.x > maxX) maxX = l.x;
|
||||
if (l.y < minY) minY = l.y;
|
||||
if (l.y > maxY) maxY = l.y;
|
||||
});
|
||||
|
||||
const w = maxX - minX;
|
||||
const h = maxY - minY;
|
||||
|
||||
const paddingX = w * 0.1;
|
||||
const paddingY = h * 0.1;
|
||||
|
||||
return {
|
||||
x: minX - paddingX,
|
||||
y: minY - paddingY,
|
||||
w: w + (paddingX * 2),
|
||||
h: h + (paddingY * 2),
|
||||
};
|
||||
};
|
||||
|
||||
const leftEyeIndices = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246];
|
||||
const rightEyeIndices = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398];
|
||||
const mouthIndices = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146];
|
||||
|
||||
const leftRect = getRect(leftEyeIndices);
|
||||
const rightRect = getRect(rightEyeIndices);
|
||||
const mouthRect = getRect(mouthIndices);
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = img.width;
|
||||
canvas.height = img.height;
|
||||
const ctx = canvas.getContext('2d');
|
||||
|
||||
let color = '#fcd3bf';
|
||||
|
||||
if (ctx) {
|
||||
ctx.drawImage(img, 0, 0);
|
||||
|
||||
const sampleIdx = 123;
|
||||
const lx = Math.floor(landmarks[sampleIdx].x * img.width);
|
||||
const ly = Math.floor(landmarks[sampleIdx].y * img.height);
|
||||
|
||||
if (lx >= 0 && lx < img.width && ly >= 0 && ly < img.height) {
|
||||
const pixel = ctx.getImageData(lx, ly, 1, 1).data;
|
||||
const toHex = (c: number) => {
|
||||
const hex = c.toString(16);
|
||||
return hex.length === 1 ? "0" + hex : hex;
|
||||
};
|
||||
color = `#${toHex(pixel[0])}${toHex(pixel[1])}${toHex(pixel[2])}`;
|
||||
}
|
||||
}
|
||||
|
||||
resolve({
|
||||
leftEye: leftRect,
|
||||
rightEye: rightRect,
|
||||
mouth: mouthRect,
|
||||
skinColor: color
|
||||
});
|
||||
} else {
|
||||
console.warn("No face detected in generated image");
|
||||
resolve(null);
|
||||
}
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
};
|
||||
img.onerror = () => reject(new Error("Failed to load image for analysis"));
|
||||
img.src = imageUrl;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Analysis failed", error);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const removeBackground = async (imageUrl: string): Promise<string> => {
|
||||
try {
|
||||
await initSegmenter();
|
||||
if (!imageSegmenter) return imageUrl;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.crossOrigin = "anonymous";
|
||||
img.onload = () => {
|
||||
try {
|
||||
const segmentResult = imageSegmenter!.segment(img);
|
||||
const confidenceMasks = segmentResult.confidenceMasks;
|
||||
|
||||
if (!confidenceMasks || confidenceMasks.length === 0) {
|
||||
resolve(imageUrl);
|
||||
return;
|
||||
}
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = img.width;
|
||||
canvas.height = img.height;
|
||||
const ctx = canvas.getContext('2d');
|
||||
|
||||
if (!ctx) {
|
||||
resolve(imageUrl);
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.drawImage(img, 0, 0);
|
||||
const imageData = ctx.getImageData(0, 0, img.width, img.height);
|
||||
const pixels = imageData.data;
|
||||
|
||||
const mask = confidenceMasks[0].getAsFloat32Array();
|
||||
|
||||
for (let i = 0; i < mask.length; i++) {
|
||||
const confidence = mask[i];
|
||||
if (confidence < 0.3) {
|
||||
pixels[i * 4 + 3] = 0; // Set Alpha to 0
|
||||
} else {
|
||||
}
|
||||
}
|
||||
|
||||
ctx.putImageData(imageData, 0, 0);
|
||||
resolve(canvas.toDataURL('image/png'));
|
||||
|
||||
} catch (e) {
|
||||
console.error("Segmentation error", e);
|
||||
resolve(imageUrl);
|
||||
}
|
||||
};
|
||||
img.onerror = () => resolve(imageUrl);
|
||||
img.src = imageUrl;
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("Background removal failed", e);
|
||||
return imageUrl;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
@import "tailwindcss";
|
||||
|
||||
/* App-level custom styles */
|
||||
body {
|
||||
margin: 0;
|
||||
background-color: #0f172a;
|
||||
color: #f8fafc;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
#root { min-height: 100vh; }
|
||||
@@ -0,0 +1,61 @@
|
||||
export enum AppState {
|
||||
SETUP = 'SETUP',
|
||||
CREATION = 'CREATION',
|
||||
RIGGING = 'RIGGING',
|
||||
STUDIO = 'STUDIO',
|
||||
}
|
||||
|
||||
export interface Rect {
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface AvatarConfig {
|
||||
imageUrl: string;
|
||||
name: string;
|
||||
description: string;
|
||||
leftEye?: Rect;
|
||||
rightEye?: Rect;
|
||||
mouth?: Rect;
|
||||
skinColor?: string;
|
||||
textureClosedEye?: Rect;
|
||||
textureOpenMouth?: Rect;
|
||||
mainBody?: Rect;
|
||||
chromaKeyColor?: string;
|
||||
}
|
||||
|
||||
export interface TrackingData {
|
||||
rotationX: number; // Pitch
|
||||
rotationY: number; // Yaw
|
||||
rotationZ: number; // Roll
|
||||
translationX: number;
|
||||
translationY: number;
|
||||
mouthOpen: number;
|
||||
isBlinkingLeft: boolean;
|
||||
isBlinkingRight: boolean;
|
||||
}
|
||||
|
||||
export interface AIStudio {
|
||||
hasSelectedApiKey(): Promise<boolean>;
|
||||
setApiKey(key: string): Promise<{ ok: boolean; error?: string; persisted?: boolean; warning?: string }>;
|
||||
clearApiKey(): Promise<{ ok: boolean; error?: string; persisted?: boolean; warning?: string }>;
|
||||
// Optional fields returned when persistence fails or succeeds
|
||||
// setApiKey may return { ok:true, persisted:false, warning:'...'} or {ok:true, persisted:true}
|
||||
// We'll model this loosely via the above return shape and consumers should check persisted/warning at runtime.
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
aistudio?: AIStudio;
|
||||
electronAPI?: {
|
||||
generateAvatar: (prompt: string, apiKey?: string) => Promise<{ image: string }>
|
||||
};
|
||||
electronLog?: {
|
||||
info: (m: string) => void;
|
||||
warn: (m: string) => void;
|
||||
error: (m: string) => void;
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user