jameshtwose/vtube-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

finally getting a dev version functioning locally... what a mess that was

Browse Source
This commit is contained in:
itsamejms 2025-11-21 22:22:53 +00:00
parent 5078d67d4f
commit ac8d171046
29 changed files with 5472 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.example Normal file
View File

@ -0,0 +1,2 @@
# Copy this file to .env and fill in your API key
GEMINI_API_KEY=your_api_key_here

1
.gitignore vendored
View File

@ -22,3 +22,4 @@ dist-ssr
*.njsproj *.njsproj
*.sln *.sln
*.sw? *.sw?
.env

74
README.md
View File

@ -1,20 +1,60 @@
<div align="center"> ## Run locally
<img width="1200" height="475" alt="GHBanner" src="https://github.com/user-attachments/assets/0aa67016-6eaf-458a-adb2-6e31a0763ed6" />
</div>
# Run and deploy your AI Studio app
This contains everything you need to run your app locally.
View your app in AI Studio: https://ai.studio/apps/drive/1Di9b15uKTFXVof4InO8oefefCDaW9Q26
## Run Locally
**Prerequisites:** Node.js
Prerequisites: Node.js (16+ recommended)
1. Install dependencies: 1. Install dependencies:
`npm install`
2. Set the `GEMINI_API_KEY` in [.env.local](.env.local) to your Gemini API key npm install
3. Run the app:
`npm run dev` 2. Copy the example env and (optionally) add your Gemini API key:
cp .env.example .env
If you set `GEMINI_API_KEY`, the project can be wired to a real image
generation backend. For local development, if the key is empty the app
will show a placeholder generated image so the UI remains usable.
3. Start the dev server:
npm run dev
## Run as an Electron app (local desktop)
1. Install dev dependencies (if not already done):
npm install
2. Start the app in Electron (dev):
npm run electron:dev
This runs Vite and then launches Electron pointed at the dev server. The renderer will use a preload bridge to ask the main process to perform avatar generation — you can extend the main process to call the `@google/genai` SDK safely.
## Tailwind CSS and production
- The app previously used the Tailwind CDN for convenience. For production we now build Tailwind locally using PostCSS (the project contains `tailwind.config.cjs` and `postcss.config.cjs`).
- To build for production (or package with Electron), run:
npm run build
- If you need to customize Tailwind, edit `tailwind.config.cjs` and your global styles in `src/styles.css`.
## Electron security note (CSP)
During development Electron warns about a permissive Content-Security-Policy (CSP). This is expected for dev, but before shipping an Electron app you must set a strict CSP and avoid `unsafe-eval` and `unsafe-inline` where possible. See: https://electronjs.org/docs/tutorial/security
---
Refactor notes
The repository was reorganized so renderer and electron-specific code live under `src/`:
- `src/renderer` — React application and browser-only services/components
- `src/electron` — main process, preload, and GenAI wrapper
- `src/shared` — types and small shared artifacts
Vite alias `@` now points to `src/` so you can import shared files as `@/shared/types`.
Compatibility: root-level electron entry files were kept as small wrappers that forward to the `src/electron` files so existing scripts should still work.
If you want further adjustments (different folders or moving more files), tell me and I will apply them.

20
index.html
View File

@ -3,9 +3,13 @@
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Gemini V-Studio</title> <title>Vtube Studio</title>
<script src="https://cdn.tailwindcss.com"></script> <!-- Tailwind is built locally via PostCSS; CSS is imported in the app entry -->
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Space+Grotesk:wght@400;700&display=swap" rel="stylesheet"> <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Space+Grotesk:wght@400;700&display=swap" rel="stylesheet">
<!-- Production CSP: tighten before packaging; development overrides via Electron main for HMR -->
<!-- Development CSP: allow jsDelivr for third-party wasm/scripts (tighten for production) -->
<!-- Development CSP: allow jsDelivr and storage.googleapis for third-party wasm/scripts and models (tighten for production) -->
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; img-src 'self' data:; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' https://fonts.gstatic.com; script-src 'self' https://cdn.jsdelivr.net 'unsafe-eval' 'unsafe-inline' data:; connect-src 'self' https://cdn.jsdelivr.net https://storage.googleapis.com https://generativelanguage.googleapis.com;">
<style> <style>
body { body {
font-family: 'Inter', sans-serif; font-family: 'Inter', sans-serif;
@ -27,19 +31,9 @@
border-radius: 4px; border-radius: 4px;
} }
</style> </style>
<script type="importmap">
{
"imports": {
"react/": "https://aistudiocdn.com/react@^19.2.0/",
"react": "https://aistudiocdn.com/react@^19.2.0",
"react-dom/": "https://aistudiocdn.com/react-dom@^19.2.0/",
"@google/genai": "https://aistudiocdn.com/@google/genai@^1.30.0",
"@mediapipe/tasks-vision": "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/+esm"
}
}
</script>
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>
<script type="module" src="/src/renderer/index.tsx"></script>
</body> </body>
</html> </html>

17
index.tsx
View File

@ -1,15 +1,2 @@
import React from 'react'; // Delegate to the new renderer entry. This file remains for compatibility with tools
import ReactDOM from 'react-dom/client'; import './src/renderer/index.tsx';
import App from './App';
const rootElement = document.getElementById('root');
if (!rootElement) {
throw new Error("Could not find root element to mount to");
}
const root = ReactDOM.createRoot(rootElement);
root.render(
<React.StrictMode>
<App />
</React.StrictMode>
);

4801
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

26
package.json
View File

@ -3,21 +3,33 @@
"private": true, "private": true,
"version": "0.0.0", "version": "0.0.0",
"type": "module", "type": "module",
"main": "src/electron/main/electron-main.cjs",
"scripts": { "scripts": {
"dev": "vite", "dev": "vite",
"build": "vite build", "build": "vite build",
"preview": "vite preview" "preview": "vite preview",
"electron:dev": "concurrently \"npm run dev\" \"wait-on http://localhost:5174 && ELECTRON_START_URL=http://localhost:5174 electron .\"",
"electron:build": "vite build && electron ."
}, },
"dependencies": { "dependencies": {
"react": "^19.2.0",
"react-dom": "^19.2.0",
"@google/genai": "^1.30.0", "@google/genai": "^1.30.0",
"@mediapipe/tasks-vision": "0.10.18" "@mediapipe/tasks-vision": "0.10.21",
"react": "^19.2.0",
"react-dom": "^19.2.0"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^22.14.0", "@types/node": "^24.10.1",
"@vitejs/plugin-react": "^5.0.0", "@vitejs/plugin-react": "^5.0.0",
"typescript": "~5.8.2", "autoprefixer": "^10.4.22",
"vite": "^6.2.0" "@tailwindcss/postcss": "^4.1.17",
"@tailwindcss/vite": "^4.1.17",
"tailwindcss": "^4.1.17",
"concurrently": "^9.2.1",
"dotenv": "^17.2.3",
"electron": "^39.2.3",
"postcss": "^8.5.6",
"typescript": "~5.9.3",
"vite": "^7.2.4",
"wait-on": "^9.0.3"
} }
} }

6
postcss.config.cjs Normal file
View File

@ -0,0 +1,6 @@
module.exports = {
// Export PostCSS plugins as an array (supported by PostCSS and Vite)
plugins: [
require('autoprefixer'),
],
};

58
services/geminiService.ts
View File

@ -1,58 +0,0 @@
import { GoogleGenAI } from "@google/genai";
/**
* Generates a VTuber avatar character sheet.
* Uses gemini-3-pro-image-preview for high quality.
*/
export const generateAvatarImage = async (description: string): Promise<string> => {
try {
// Initialize client inside the function to ensure we use the most up-to-date API key
const ai = new GoogleGenAI({ apiKey: process.env.API_KEY });
const prompt = `
Create a VTuber character sheet with a flat 2D anime style.
LAYOUT:
1. MAIN CHARACTER (Left side, takes up 70% of width):
- Front-facing view, head and shoulders.
- Neutral expression, eyes open, mouth closed.
2. EXPRESSION ASSETS (Right side, vertical column):
- Top: The same character's face with EYES CLOSED (for blinking).
- Bottom: The same character's face with MOUTH OPEN (for talking).
Character Description: ${description}
Style: Vibrant, clean lines, solid white or green background for easy keying.
`;
const response = await ai.models.generateContent({
model: 'gemini-3-pro-image-preview',
contents: {
parts: [
{ text: prompt }
]
},
config: {
imageConfig: {
aspectRatio: "16:9", // Wide to fit character sheet
imageSize: "1K"
}
}
});
// Parse response for image data
for (const part of response.candidates[0].content.parts) {
if (part.inlineData) {
const base64EncodeString = part.inlineData.data;
return `data:image/png;base64,${base64EncodeString}`;
}
}
throw new Error("No image data found in response");
} catch (error) {
console.error("Error generating avatar:", error);
throw error;
}
};

26
src/electron/main/electron-main.cjs Normal file
View File

@ -0,0 +1,26 @@
(async () => {
try {
// In dev, load a local .env so ENV values like GEMINI_API_KEY are available to the main process.
try {
// eslint-disable-next-line no-var
var dotenv = await import('dotenv');
dotenv.config();
console.log('[electron-main.cjs] Loaded .env into process.env');
} catch (e) {
// Not fatal; dotenv may not be installed in some environments
console.log('[electron-main.cjs] dotenv not available, skipping .env load');
}
const mod = await import('./electron-main.js');
const { ipcMain } = await import('electron');
ipcMain.on('renderer-log', (event, { level, msg }) => {
const prefix = `[renderer ${level}]`;
if (level === 'error') console.error(prefix, msg);
else if (level === 'warn') console.warn(prefix, msg);
else console.log(prefix, msg);
});
} catch (e) {
console.error('Failed to load ESM electron main:', e);
process.exit(1);
}
})();

103
src/electron/main/electron-main.js Normal file
View File

@ -0,0 +1,103 @@
import { app, BrowserWindow, ipcMain } from 'electron';
import path from 'path';
import url from 'url';
import { fileURLToPath } from 'url';
// Keep an in-memory API key for the running session only. Renderer should still store key in localStorage.
let inMemoryKey = null;
function createWindow() {
// Resolve dirname equivalent in ESM
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const win = new BrowserWindow({
width: 1200,
height: 800,
webPreferences: {
// Preload is colocated with the electron main files after the refactor
preload: path.join(__dirname, 'electron-preload.js'),
contextIsolation: true,
nodeIntegration: false,
}
});
const startUrl = process.env.ELECTRON_START_URL || url.pathToFileURL(path.join(process.cwd(), 'dist', 'index.html')).toString();
win.loadURL(startUrl).catch(err => {
console.error('[electron-main] Failed to load URL:', err);
});
if (process.env.ELECTRON_START_URL) {
try {
const ses = win.webContents.session;
ses.webRequest.onHeadersReceived((details, callback) => {
const headers = details.responseHeaders || {};
// Allow jsDelivr CDN for scripts used by third-party libs (dev only)
// Dev CSP: allow inline scripts, unsafe-eval and jsDelivr CDN for third-party libs (HMR and wasm loaders need inline scripts/eval)
headers['Content-Security-Policy'] = [
"default-src 'self' 'unsafe-eval' 'unsafe-inline' data:; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.jsdelivr.net; img-src 'self' data:; connect-src *; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src 'self' data: https://fonts.gstatic.com"
];
callback({ responseHeaders: headers });
});
} catch (e) {
console.warn('[electron-main] Failed to inject dev CSP:', e);
}
win.webContents.once('did-frame-finish-load', () => {
try {
win.webContents.openDevTools({ mode: 'right' });
} catch (e) {
console.warn('[electron-main] Could not open DevTools:', e);
}
});
}
win.webContents.on('did-finish-load', () => {
console.log('[electron-main] Renderer finished load; title=', win.getTitle());
});
}
app.whenReady().then(() => {
createWindow();
app.on('activate', function () {
if (BrowserWindow.getAllWindows().length === 0) createWindow();
});
});
app.on('window-all-closed', function () {
if (process.platform !== 'darwin') app.quit();
});
ipcMain.handle('generate-avatar', async (event, prompt) => {
try {
console.log('[electron-main] generate-avatar handler invoked');
console.log('[electron-main] prompt length:', (prompt || '').length);
// Prefer an in-memory key, then environment variables
const apiKey = inMemoryKey || process.env.GEMINI_API_KEY || process.env.API_KEY;
console.log('[electron-main] apiKey present?', !!apiKey, '(will prefer GEMINI_API_KEY if set)');
if (apiKey) {
try {
console.log('[electron-main] Calling GenAI helper...');
const imageData = await generateAvatarWithGenAI(prompt || '', apiKey);
console.log('[electron-main] GenAI helper returned image, length:', imageData?.length || 0);
return { image: imageData };
} catch (e) {
console.error('[electron-main] GenAI generation failed:', e?.message || e);
}
} else {
console.log('[electron-main] No API key present — skipping GenAI call and returning placeholder');
}
} catch (outerErr) {
console.error('[electron-main] Unexpected error in generate-avatar handler:', outerErr);
}
// Fallback placeholder image
const placeholder = `data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='1400' height='900'><rect width='100%' height='100%' fill='%230f172a' /><text x='50%' y='50%' fill='white' font-size='40' font-family='Inter' dominant-baseline='middle' text-anchor='middle'>Placeholder: ${encodeURIComponent((prompt||'').substring(0,80))}</text></svg>`;
console.log('[electron-main] Returning placeholder image (length):', placeholder.length);
return { image: placeholder };
});
// Expose simple key management for the renderer via ipc (session-only)
// No renderer-side key persistence — frontend uses localStorage exclusively now.

45
src/electron/preload/electron-preload.js Normal file
View File

@ -0,0 +1,45 @@
const { contextBridge, ipcRenderer } = require('electron');
contextBridge.exposeInMainWorld('electronAPI', {
// Accept an optional apiKey argument so the renderer can pass a stored key from localStorage
generateAvatar: (prompt, apiKey) => ipcRenderer.invoke('generate-avatar', prompt, apiKey)
});
contextBridge.exposeInMainWorld('electronLog', {
info: (msg) => ipcRenderer.send('renderer-log', { level: 'info', msg }),
warn: (msg) => ipcRenderer.send('renderer-log', { level: 'warn', msg }),
error: (msg) => ipcRenderer.send('renderer-log', { level: 'error', msg }),
});
try {
ipcRenderer.send('renderer-log', { level: 'info', msg: 'preload loaded' });
} catch (e) {
}
try {
const forward = (level, args) => {
try {
const msg = args.map(a => {
try { return typeof a === 'string' ? a : JSON.stringify(a); } catch (e) { return String(a); }
}).join(' ');
ipcRenderer.send('renderer-log', { level, msg });
} catch (e) { }
};
const origError = console.error.bind(console);
console.error = (...args) => { forward('error', args); origError(...args); };
const origWarn = console.warn.bind(console);
console.warn = (...args) => { forward('warn', args); origWarn(...args); };
const origLog = console.log.bind(console);
console.log = (...args) => { forward('info', args); origLog(...args); };
// DOM snapshot logging removed — keep console forwarding only.
window.addEventListener('error', (ev) => {
try { ipcRenderer.send('renderer-log', { level: 'error', msg: `window.onerror: ${ev.message} ${ev.filename}:${ev.lineno}:${ev.colno}` }); } catch (e) {}
});
} catch (e) {
}

91
App.tsx → src/renderer/App.tsx
View File

@ -1,32 +1,77 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { AppState, AvatarConfig, Rect } from './types'; import { AppState, AvatarConfig, Rect } from '../shared/types';
import AvatarCreator from './components/AvatarCreator'; import AvatarCreator from './components/AvatarCreator';
import RiggingEditor from './components/RiggingEditor'; import RiggingEditor from './components/RiggingEditor';
import Studio from './components/Studio'; import Studio from './components/Studio';
const App: React.FC = () => { const App: React.FC = () => {
const [appState, setAppState] = useState<AppState>(AppState.SETUP); const [appState, setAppState] = useState<AppState>(AppState.SETUP);
// Temp storage for the generated image before rigging
const [generatedData, setGeneratedData] = useState<{url: string, name: string, initialData?: any} | null>(null); const [generatedData, setGeneratedData] = useState<{url: string, name: string, initialData?: any} | null>(null);
const [avatar, setAvatar] = useState<AvatarConfig | null>(null); const [avatar, setAvatar] = useState<AvatarConfig | null>(null);
const handleStartCreation = async () => { const handleStartCreation = async () => {
try { try {
if (window.aistudio) { if ((window as any).electronLog) (window as any).electronLog.info('handleStartCreation called');
const hasKey = await window.aistudio.hasSelectedApiKey(); // No blocking API key flow — we just try to ensure a key is set for better UX
if (!hasKey) {
await window.aistudio.openSelectKey();
}
}
setAppState(AppState.CREATION); setAppState(AppState.CREATION);
} catch (error) { } catch (error) {
console.error("Error during API key selection:", error); console.error("Error during API key selection:", error);
if ((window as any).electronLog) (window as any).electronLog.error(`API key selection failed: ${String(error)}`);
setAppState(AppState.CREATION); setAppState(AppState.CREATION);
} }
}; };
const [hasKey, setHasKey] = useState<boolean>(false);
const [showKeyModal, setShowKeyModal] = React.useState(false);
const [keyInput, setKeyInput] = React.useState('');
const refreshKeyStatus = async () => {
try {
const present = !!(typeof window !== 'undefined' && localStorage.getItem('GEMINI_API_KEY'));
setHasKey(present);
} catch (e) {
console.warn('Failed to check API key status from localStorage', e);
setHasKey(false);
}
};
React.useEffect(() => {
refreshKeyStatus();
}, []);
const openKeyModal = () => {
setKeyInput('');
setShowKeyModal(true);
};
const submitKey = async () => {
try {
if (!keyInput) return;
// Only store in localStorage (renderer-side). Do not attempt IPC or disk persistence.
try { localStorage.setItem('GEMINI_API_KEY', keyInput); } catch (e) { console.warn('Failed to write key to localStorage', e); alert('Failed to save key to localStorage'); return; }
setShowKeyModal(false);
refreshKeyStatus();
try { window.electronLog?.info('API key saved to localStorage'); } catch {}
alert('API key saved to localStorage');
} catch (e) {
console.error(e);
alert('Error saving key');
}
};
const handleClearKey = async () => {
try {
try { localStorage.removeItem('GEMINI_API_KEY'); } catch (e) { console.warn('Failed to remove key from localStorage', e); alert('Failed to clear key from localStorage'); return; }
refreshKeyStatus();
alert('API key cleared from localStorage');
} catch (e) {
console.error(e);
}
};
const handleAvatarGenerated = (url: string, name: string, initialData?: any) => { const handleAvatarGenerated = (url: string, name: string, initialData?: any) => {
if ((window as any).electronLog) (window as any).electronLog.info(`avatar generated: ${name}`);
setGeneratedData({ url, name, initialData }); setGeneratedData({ url, name, initialData });
setAppState(AppState.RIGGING); setAppState(AppState.RIGGING);
}; };
@ -55,6 +100,7 @@ const App: React.FC = () => {
}; };
return ( return (
<>
<div className="min-h-screen bg-slate-900 text-white"> <div className="min-h-screen bg-slate-900 text-white">
{appState === AppState.SETUP && ( {appState === AppState.SETUP && (
<div className="container mx-auto px-4 py-12 flex flex-col items-center justify-center min-h-screen"> <div className="container mx-auto px-4 py-12 flex flex-col items-center justify-center min-h-screen">
@ -71,6 +117,11 @@ const App: React.FC = () => {
> >
Start Creation Start Creation
</button> </button>
<div className="mt-4 flex items-center justify-center gap-3">
<button onClick={openKeyModal} className="px-4 py-2 rounded bg-slate-700 hover:bg-slate-600 text-sm">Set API Key</button>
<button onClick={handleClearKey} className="px-4 py-2 rounded bg-red-600 hover:bg-red-500 text-sm">Clear Key</button>
<div className="text-sm text-slate-400">Key: {hasKey ? <span className="text-green-400">Saved</span> : <span className="text-rose-400">Not set</span>}</div>
</div>
</div> </div>
<div className="grid grid-cols-1 md:grid-cols-3 gap-8 w-full max-w-5xl"> <div className="grid grid-cols-1 md:grid-cols-3 gap-8 w-full max-w-5xl">
@ -129,7 +180,29 @@ const App: React.FC = () => {
onBack={() => setAppState(AppState.SETUP)} onBack={() => setAppState(AppState.SETUP)}
/> />
)} )}
{/* TailwindDebug removed */}
</div> </div>
{showKeyModal && (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60">
<div className="bg-slate-900 rounded-xl p-6 w-full max-w-lg">
<h3 className="text-lg font-bold mb-3">Enter Gemini API Key</h3>
<p className="text-sm text-slate-400 mb-4">This key will be stored locally in the app data directory.</p>
<input
type="password"
value={keyInput}
onChange={(e) => setKeyInput(e.target.value)}
className="w-full bg-slate-800 border border-slate-700 rounded px-3 py-2 text-white mb-4"
placeholder="sk_..."
/>
<div className="flex justify-end gap-3">
<button onClick={() => setShowKeyModal(false)} className="px-4 py-2 rounded bg-slate-700 hover:bg-slate-600">Cancel</button>
<button onClick={submitKey} className="px-4 py-2 rounded bg-cyan-500 hover:bg-cyan-400 text-slate-900 font-bold">Save</button>
</div>
</div>
</div>
)}
</>
); );
}; };

29
components/AvatarCreator.tsx → src/renderer/components/AvatarCreator.tsx
View File

@ -1,9 +1,14 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { generateAvatarImage } from '../services/geminiService';
import { analyzeAvatarImage } from '../services/visionService'; import { analyzeAvatarImage } from '../services/visionService';
import { stitchAssets, fileToDataUrl } from '../services/imageService'; import { stitchAssets, fileToDataUrl } from '../services/imageService';
import { generateAvatarImage } from '../services/geminiService';
import LoadingSpinner from './LoadingSpinner'; import LoadingSpinner from './LoadingSpinner';
import { Rect } from '../types'; import { Rect } from '../../shared/types';
const placeholderGenerate = async (prompt: string) => {
const text = encodeURIComponent((prompt || '').substring(0, 80));
return `data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='1400' height='900'><rect width='100%' height='100%' fill='%230f172a' /><text x='50%' y='50%' fill='white' font-size='40' font-family='Inter' dominant-baseline='middle' text-anchor='middle'>Placeholder: ${text}</text></svg>`;
};
interface AvatarCreatorProps { interface AvatarCreatorProps {
onAvatarGenerated: (url: string, name: string, initialData?: { onAvatarGenerated: (url: string, name: string, initialData?: {
@ -15,13 +20,11 @@ interface AvatarCreatorProps {
const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => { const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
const [mode, setMode] = useState<'generate' | 'upload'>('generate'); const [mode, setMode] = useState<'generate' | 'upload'>('generate');
// Generation State
const [prompt, setPrompt] = useState(''); const [prompt, setPrompt] = useState('');
const [name, setName] = useState(''); const [name, setName] = useState('');
const [status, setStatus] = useState<'idle' | 'generating' | 'analyzing' | 'stitching'>('idle'); const [status, setStatus] = useState<'idle' | 'generating' | 'analyzing' | 'stitching'>('idle');
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
// Upload State
const [baseFile, setBaseFile] = useState<File | null>(null); const [baseFile, setBaseFile] = useState<File | null>(null);
const [blinkFile, setBlinkFile] = useState<File | null>(null); const [blinkFile, setBlinkFile] = useState<File | null>(null);
const [talkFile, setTalkFile] = useState<File | null>(null); const [talkFile, setTalkFile] = useState<File | null>(null);
@ -33,14 +36,11 @@ const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
setError(null); setError(null);
try { try {
// 1. Generate Image (Now creates a character sheet)
const imageUrl = await generateAvatarImage(prompt); const imageUrl = await generateAvatarImage(prompt);
// 2. Analyze Image for Landmarks
setStatus('analyzing'); setStatus('analyzing');
const analysisData = await analyzeAvatarImage(imageUrl); const analysisData = await analyzeAvatarImage(imageUrl);
// 3. Pass to parent
if (analysisData) { if (analysisData) {
onAvatarGenerated(imageUrl, name, analysisData); onAvatarGenerated(imageUrl, name, analysisData);
} else { } else {
@ -61,33 +61,23 @@ const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
setError(null); setError(null);
try { try {
// 1. Prepare Base Image and Analyze it separately
// Analyzing separately ensures we get landmarks for the main face correctly
// without interference from other faces in a stitched sheet.
const baseDataUrl = await fileToDataUrl(baseFile); const baseDataUrl = await fileToDataUrl(baseFile);
const baseAnalysis = await analyzeAvatarImage(baseDataUrl); const baseAnalysis = await analyzeAvatarImage(baseDataUrl);
// 2. Prepare and Analyze Variant Images
let blinkDataUrl, blinkAnalysis; let blinkDataUrl, blinkAnalysis;
if (blinkFile) { if (blinkFile) {
blinkDataUrl = await fileToDataUrl(blinkFile); blinkDataUrl = await fileToDataUrl(blinkFile);
// Try to find eyes in the blink image to use as tight texture crop
blinkAnalysis = await analyzeAvatarImage(blinkDataUrl); blinkAnalysis = await analyzeAvatarImage(blinkDataUrl);
} }
let talkDataUrl, talkAnalysis; let talkDataUrl, talkAnalysis;
if (talkFile) { if (talkFile) {
talkDataUrl = await fileToDataUrl(talkFile); talkDataUrl = await fileToDataUrl(talkFile);
// Try to find mouth in the talk image
talkAnalysis = await analyzeAvatarImage(talkDataUrl); talkAnalysis = await analyzeAvatarImage(talkDataUrl);
} }
// 3. Stitch Assets into Sheet
const { imageUrl, mainBody, textureClosedEye: stitchBlinkRect, textureOpenMouth: stitchTalkRect } = await stitchAssets(baseDataUrl, blinkDataUrl, talkDataUrl); const { imageUrl, mainBody, textureClosedEye: stitchBlinkRect, textureOpenMouth: stitchTalkRect } = await stitchAssets(baseDataUrl, blinkDataUrl, talkDataUrl);
// 4. Map Analysis Data to Stitched Coordinate Space
// Helper to map a rect from (0-1 in sub-image) to (0-1 in stitched-image)
const mapRect = (r: Rect, container: Rect) => ({ const mapRect = (r: Rect, container: Rect) => ({
x: container.x + r.x * container.w, x: container.x + r.x * container.w,
y: container.y + r.y * container.h, y: container.y + r.y * container.h,
@ -101,7 +91,6 @@ const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
textureOpenMouth: stitchTalkRect textureOpenMouth: stitchTalkRect
}; };
// Map Base Targets (Eyes, Mouth on main body)
if (baseAnalysis) { if (baseAnalysis) {
initialData.leftEye = mapRect(baseAnalysis.leftEye, mainBody); initialData.leftEye = mapRect(baseAnalysis.leftEye, mainBody);
initialData.rightEye = mapRect(baseAnalysis.rightEye, mainBody); initialData.rightEye = mapRect(baseAnalysis.rightEye, mainBody);
@ -109,10 +98,7 @@ const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
initialData.skinColor = baseAnalysis.skinColor; initialData.skinColor = baseAnalysis.skinColor;
} }
// Map Source Textures (Tight crop around features if detected)
// If detections fail (e.g. eyes closed might not be detected), we fall back to the whole image (stitchBlinkRect)
if (blinkAnalysis && stitchBlinkRect) { if (blinkAnalysis && stitchBlinkRect) {
// Calculate a bounding box around both eyes in the blink image
const be = blinkAnalysis; const be = blinkAnalysis;
const minX = Math.min(be.leftEye.x, be.rightEye.x); const minX = Math.min(be.leftEye.x, be.rightEye.x);
const minY = Math.min(be.leftEye.y, be.rightEye.y); const minY = Math.min(be.leftEye.y, be.rightEye.y);
@ -144,7 +130,6 @@ const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
return ( return (
<div className="max-w-2xl mx-auto bg-slate-800/50 backdrop-blur-lg border border-slate-700 rounded-2xl shadow-2xl overflow-hidden"> <div className="max-w-2xl mx-auto bg-slate-800/50 backdrop-blur-lg border border-slate-700 rounded-2xl shadow-2xl overflow-hidden">
{/* Tabs */}
<div className="flex border-b border-slate-700"> <div className="flex border-b border-slate-700">
<button <button
onClick={() => setMode('generate')} onClick={() => setMode('generate')}

16
src/renderer/components/ErrorBoundary.tsx Normal file
View File

@ -0,0 +1,16 @@
import React from 'react';
const ErrorBoundaryFallback: React.FC<{error?: Error | null}> = ({ error }) => (
<div style={{ padding: 24, color: 'white' }}>
<h2>Something went wrong</h2>
<pre style={{ whiteSpace: 'pre-wrap' }}>{String(error)}</pre>
</div>
);
// Lightweight functional error boundary placeholder. Replace with a full class-based boundary
// if you need to catch render-time errors.
const ErrorBoundary: React.FC<{children: React.ReactNode}> = ({ children }) => {
return <>{children}</>;
};
export default ErrorBoundary;

0
components/LoadingSpinner.tsx → src/renderer/components/LoadingSpinner.tsx
View File

15
components/RiggingEditor.tsx → src/renderer/components/RiggingEditor.tsx
View File

@ -1,6 +1,5 @@
import React, { useState, useRef, useEffect } from 'react'; import React, { useState, useRef, useEffect } from 'react';
import { Rect } from '../types'; import { Rect } from '../../shared/types';
interface RiggingEditorProps { interface RiggingEditorProps {
imageUrl: string; imageUrl: string;
@ -102,7 +101,6 @@ const ResizableBox: React.FC<{
backgroundColor: isActive ? `${color}20` : 'transparent', backgroundColor: isActive ? `${color}20` : 'transparent',
}} }}
> >
{/* Label */}
<div <div
className="absolute -top-6 left-0 text-xs font-bold px-1 rounded text-white whitespace-nowrap shadow-sm" className="absolute -top-6 left-0 text-xs font-bold px-1 rounded text-white whitespace-nowrap shadow-sm"
style={{ backgroundColor: color }} style={{ backgroundColor: color }}
@ -110,7 +108,6 @@ const ResizableBox: React.FC<{
{label} {label}
</div> </div>
{/* Resize Handle */}
<div <div
onMouseDown={handleResizeDown} onMouseDown={handleResizeDown}
className="absolute bottom-0 right-0 w-4 h-4 bg-white border-2 cursor-nwse-resize opacity-0 group-hover:opacity-100 transition-opacity" className="absolute bottom-0 right-0 w-4 h-4 bg-white border-2 cursor-nwse-resize opacity-0 group-hover:opacity-100 transition-opacity"
@ -121,20 +118,16 @@ const ResizableBox: React.FC<{
}; };
const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, onComplete }) => { const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, onComplete }) => {
// Targets (Left side of image usually)
const [leftEye, setLeftEye] = useState<Rect>(initialData?.leftEye || { x: 0.25, y: 0.4, w: 0.1, h: 0.1 }); const [leftEye, setLeftEye] = useState<Rect>(initialData?.leftEye || { x: 0.25, y: 0.4, w: 0.1, h: 0.1 });
const [rightEye, setRightEye] = useState<Rect>(initialData?.rightEye || { x: 0.45, y: 0.4, w: 0.1, h: 0.1 }); const [rightEye, setRightEye] = useState<Rect>(initialData?.rightEye || { x: 0.45, y: 0.4, w: 0.1, h: 0.1 });
const [mouth, setMouth] = useState<Rect>(initialData?.mouth || { x: 0.35, y: 0.55, w: 0.1, h: 0.05 }); const [mouth, setMouth] = useState<Rect>(initialData?.mouth || { x: 0.35, y: 0.55, w: 0.1, h: 0.05 });
// Main Body (Default to left 70%)
const [mainBody, setMainBody] = useState<Rect>({ x: 0.05, y: 0.05, w: 0.65, h: 0.9 }); const [mainBody, setMainBody] = useState<Rect>({ x: 0.05, y: 0.05, w: 0.65, h: 0.9 });
// Sources (Right side of image usually)
const [textureClosedEye, setTextureClosedEye] = useState<Rect>({ x: 0.7, y: 0.1, w: 0.2, h: 0.2 }); const [textureClosedEye, setTextureClosedEye] = useState<Rect>({ x: 0.7, y: 0.1, w: 0.2, h: 0.2 });
const [textureOpenMouth, setTextureOpenMouth] = useState<Rect>({ x: 0.7, y: 0.5, w: 0.2, h: 0.2 }); const [textureOpenMouth, setTextureOpenMouth] = useState<Rect>({ x: 0.7, y: 0.5, w: 0.2, h: 0.2 });
const [skinColor, setSkinColor] = useState<string>(initialData?.skinColor || '#fcd3bf'); const [skinColor, setSkinColor] = useState<string>(initialData?.skinColor || '#fcd3bf');
// Use this simply as a boolean flag now, passing 'AI_AUTO' if enabled
const [useAiBackground, setUseAiBackground] = useState<boolean>(true); const [useAiBackground, setUseAiBackground] = useState<boolean>(true);
const [activeFeature, setActiveFeature] = useState<ActiveFeature>(null); const [activeFeature, setActiveFeature] = useState<ActiveFeature>(null);
@ -151,7 +144,6 @@ const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, on
</div> </div>
<div className="flex gap-6 w-full items-start h-[70vh]"> <div className="flex gap-6 w-full items-start h-[70vh]">
{/* Editor Area */}
<div className="flex-1 bg-slate-800 p-4 rounded-xl border border-slate-700 flex justify-center h-full overflow-hidden relative"> <div className="flex-1 bg-slate-800 p-4 rounded-xl border border-slate-700 flex justify-center h-full overflow-hidden relative">
<div className="relative inline-block h-full"> <div className="relative inline-block h-full">
<img <img
@ -161,16 +153,13 @@ const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, on
draggable={false} draggable={false}
/> />
{/* Aspect ratio container to map percentage boxes correctly */}
<div className="absolute inset-0 w-full h-full"> <div className="absolute inset-0 w-full h-full">
{/* Main Body */}
<ResizableBox <ResizableBox
rect={mainBody} color="#facc15" label="Main Body" rect={mainBody} color="#facc15" label="Main Body"
isActive={activeFeature === 'mainBody'} isActive={activeFeature === 'mainBody'}
onUpdate={setMainBody} onActivate={() => setActiveFeature('mainBody')} onUpdate={setMainBody} onActivate={() => setActiveFeature('mainBody')}
/> />
{/* Targets */}
<ResizableBox <ResizableBox
rect={leftEye} color="#ef4444" label="Left Eye Target" rect={leftEye} color="#ef4444" label="Left Eye Target"
isActive={activeFeature === 'leftEye'} isActive={activeFeature === 'leftEye'}
@ -187,7 +176,6 @@ const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, on
onUpdate={setMouth} onActivate={() => setActiveFeature('mouth')} onUpdate={setMouth} onActivate={() => setActiveFeature('mouth')}
/> />
{/* Sources */}
<ResizableBox <ResizableBox
rect={textureClosedEye} color="#a855f7" label="Source: Closed Eyes" rect={textureClosedEye} color="#a855f7" label="Source: Closed Eyes"
isActive={activeFeature === 'textureClosedEye'} isActive={activeFeature === 'textureClosedEye'}
@ -202,7 +190,6 @@ const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, on
</div> </div>
</div> </div>
{/* Sidebar Controls */}
<div className="w-72 flex flex-col gap-4 bg-slate-800/50 p-6 rounded-xl border border-slate-700 h-full overflow-y-auto"> <div className="w-72 flex flex-col gap-4 bg-slate-800/50 p-6 rounded-xl border border-slate-700 h-full overflow-y-auto">
<div className="bg-slate-900/50 p-4 rounded-lg space-y-3"> <div className="bg-slate-900/50 p-4 rounded-lg space-y-3">

50
components/Studio.tsx → src/renderer/components/Studio.tsx
View File

@ -1,8 +1,7 @@
import React, { useEffect, useRef, useState } from 'react'; import React, { useEffect, useRef, useState } from 'react';
import { useFaceTracking } from '../hooks/useFaceTracking'; import { useFaceTracking } from '../hooks/useFaceTracking';
import { removeBackground } from '../services/visionService'; import { removeBackground } from '../services/visionService';
import { AvatarConfig, Rect } from '../types'; import { AvatarConfig, Rect } from '../../shared/types';
import LoadingSpinner from './LoadingSpinner'; import LoadingSpinner from './LoadingSpinner';
interface StudioProps { interface StudioProps {
@ -10,19 +9,12 @@ interface StudioProps {
onBack: () => void; onBack: () => void;
} }
/**
* Sprite Component
* Renders a specific crop of the source image into a target container.
*/
const Sprite: React.FC<{ const Sprite: React.FC<{
imageSrc: string; imageSrc: string;
sourceRect: Rect; sourceRect: Rect;
style?: React.CSSProperties; style?: React.CSSProperties;
className?: string; className?: string;
}> = ({ imageSrc, sourceRect, style, className }) => { }> = ({ imageSrc, sourceRect, style, className }) => {
// To display a cropped region (sourceRect) of the image, we use an inner <img>
// positioned negatively and scaled up.
// Example: If sourceRect.w is 0.1 (10%), the image must be scaled to 10x (1000%) size.
const widthScale = 100 / (sourceRect.w * 100); const widthScale = 100 / (sourceRect.w * 100);
const heightScale = 100 / (sourceRect.h * 100); const heightScale = 100 / (sourceRect.h * 100);
@ -54,33 +46,32 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
const [cameraReady, setCameraReady] = useState(false); const [cameraReady, setCameraReady] = useState(false);
const [processedImageUrl, setProcessedImageUrl] = useState<string | null>(null); const [processedImageUrl, setProcessedImageUrl] = useState<string | null>(null);
// We use the custom hook to get tracking data
const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current); const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current);
// Initialize Camera
useEffect(() => { useEffect(() => {
const startCamera = async () => { const startCamera = async () => {
try { try {
const stream = await navigator.mediaDevices.getUserMedia({ const stream = await navigator.mediaDevices.getUserMedia({
video: { width: 640, height: 480 }, // Lower res is fine for tracking video: { width: 640, height: 480 },
audio: false audio: false
}); });
if (videoRef.current) { if (videoRef.current) {
videoRef.current.srcObject = stream; videoRef.current.srcObject = stream;
videoRef.current.onloadeddata = () => { videoRef.current.onloadeddata = () => {
setCameraReady(true); setCameraReady(true);
if ((window as any).electronLog) (window as any).electronLog.info('Camera ready');
}; };
} }
} catch (err) { } catch (err) {
console.error("Error accessing camera:", err); console.error("Error accessing camera:", err);
alert("Could not access camera. Please ensure permissions are granted."); alert("Could not access camera. Please ensure permissions are granted.");
if ((window as any).electronLog) (window as any).electronLog.error(`Camera access failed: ${String(err)}`);
} }
}; };
startCamera(); startCamera();
return () => { return () => {
// Cleanup stream
if (videoRef.current && videoRef.current.srcObject) { if (videoRef.current && videoRef.current.srcObject) {
const stream = videoRef.current.srcObject as MediaStream; const stream = videoRef.current.srcObject as MediaStream;
stream.getTracks().forEach(track => track.stop()); stream.getTracks().forEach(track => track.stop());
@ -88,7 +79,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
}; };
}, []); }, []);
// Process Image for Background Removal (AI Segmentation)
useEffect(() => { useEffect(() => {
if (!avatar.chromaKeyColor) { if (!avatar.chromaKeyColor) {
setProcessedImageUrl(avatar.imageUrl); setProcessedImageUrl(avatar.imageUrl);
@ -96,7 +86,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
} }
const process = async () => { const process = async () => {
// If chromaKeyColor is set (to anything, now treated as a flag), we run AI removal
const result = await removeBackground(avatar.imageUrl); const result = await removeBackground(avatar.imageUrl);
setProcessedImageUrl(result); setProcessedImageUrl(result);
}; };
@ -104,25 +93,21 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
process(); process();
}, [avatar.imageUrl, avatar.chromaKeyColor]); }, [avatar.imageUrl, avatar.chromaKeyColor]);
// Start tracking when both camera and model are ready
useEffect(() => { useEffect(() => {
if (cameraReady && !isModelLoading) { if (cameraReady && !isModelLoading) {
startTracking(); startTracking();
} }
}, [cameraReady, isModelLoading, startTracking]); }, [cameraReady, isModelLoading, startTracking]);
// Calculate styles based on tracking data
const getAvatarStyle = () => { const getAvatarStyle = () => {
// Deadzone for jitter reduction
const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val; const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val;
const rX = smooth(trackingData.rotationX); // Pitch const rX = smooth(trackingData.rotationX);
const rY = smooth(trackingData.rotationY); // Yaw const rY = smooth(trackingData.rotationY);
const rZ = smooth(trackingData.rotationZ); // Roll const rZ = smooth(trackingData.rotationZ);
const tX = smooth(trackingData.translationX); const tX = smooth(trackingData.translationX);
const tY = smooth(trackingData.translationY); const tY = smooth(trackingData.translationY);
// Bounce effect on mouth open (Speaking emulation)
const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0; const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0;
return { return {
@ -134,14 +119,13 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
rotateY(${rY * -25}deg) rotateY(${rY * -25}deg)
scale(${1 + trackingData.mouthOpen * 0.02}) scale(${1 + trackingData.mouthOpen * 0.02})
`, `,
filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`, // Slight flash when speaking filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`,
transition: 'transform 0.1s ease-out, filter 0.1s ease' transition: 'transform 0.1s ease-out, filter 0.1s ease'
}; } as React.CSSProperties;
}; };
return ( return (
<div className="h-screen w-full flex flex-col bg-slate-900 overflow-hidden relative"> <div className="h-screen w-full flex flex-col bg-slate-900 overflow-hidden relative">
{/* Hidden Video Element for Tracking */}
<video <video
ref={videoRef} ref={videoRef}
autoPlay autoPlay
@ -150,7 +134,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
className="absolute opacity-0 pointer-events-none w-1 h-1" className="absolute opacity-0 pointer-events-none w-1 h-1"
/> />
{/* Top Bar */}
<div className="absolute top-0 left-0 right-0 z-20 p-4 flex justify-between items-center bg-gradient-to-b from-slate-900 to-transparent"> <div className="absolute top-0 left-0 right-0 z-20 p-4 flex justify-between items-center bg-gradient-to-b from-slate-900 to-transparent">
<button <button
onClick={onBack} onClick={onBack}
@ -169,9 +152,7 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
</div> </div>
</div> </div>
{/* Main Stage */}
<div className="flex-1 relative flex items-center justify-center overflow-hidden"> <div className="flex-1 relative flex items-center justify-center overflow-hidden">
{/* Background Grid/Effect */}
<div className="absolute inset-0 opacity-20" <div className="absolute inset-0 opacity-20"
style={{ style={{
backgroundImage: 'radial-gradient(#4f46e5 1px, transparent 1px)', backgroundImage: 'radial-gradient(#4f46e5 1px, transparent 1px)',
@ -181,7 +162,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
<div className="absolute inset-0 bg-gradient-to-t from-slate-900 via-transparent to-slate-900 pointer-events-none"></div> <div className="absolute inset-0 bg-gradient-to-t from-slate-900 via-transparent to-slate-900 pointer-events-none"></div>
{/* Avatar Container */}
<div className="relative w-[600px] h-[600px] flex items-center justify-center z-10"> <div className="relative w-[600px] h-[600px] flex items-center justify-center z-10">
{!processedImageUrl ? ( {!processedImageUrl ? (
<div className="flex flex-col items-center justify-center gap-4"> <div className="flex flex-col items-center justify-center gap-4">
@ -193,7 +173,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
className="relative w-full h-full flex items-center justify-center" className="relative w-full h-full flex items-center justify-center"
style={getAvatarStyle()} style={getAvatarStyle()}
> >
{/* Main Character Body (Cropped using Sprite) */}
{avatar.mainBody ? ( {avatar.mainBody ? (
<Sprite <Sprite
imageSrc={processedImageUrl} imageSrc={processedImageUrl}
@ -201,7 +180,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]" className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
/> />
) : ( ) : (
/* Fallback to full image if mainBody is missing */
<img <img
src={processedImageUrl} src={processedImageUrl}
alt="Avatar" alt="Avatar"
@ -209,7 +187,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
/> />
)} )}
{/* Dynamic Eyelids (High Fidelity Sprites) */}
{avatar.leftEye && avatar.textureClosedEye && ( {avatar.leftEye && avatar.textureClosedEye && (
<Sprite <Sprite
imageSrc={processedImageUrl} imageSrc={processedImageUrl}
@ -242,7 +219,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
/> />
)} )}
{/* Dynamic Mouth Animation */}
{avatar.mouth && avatar.textureOpenMouth && ( {avatar.mouth && avatar.textureOpenMouth && (
<div <div
className="absolute pointer-events-none flex items-center justify-center z-10" className="absolute pointer-events-none flex items-center justify-center z-10"
@ -253,25 +229,22 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
height: `${avatar.mouth.h * 100}%`, height: `${avatar.mouth.h * 100}%`,
}} }}
> >
{/* Skin Patch - Hides the static closed mouth when speaking */}
<div <div
className="absolute w-[120%] h-[120%] transition-opacity duration-75" className="absolute w-[120%] h-[120%] transition-opacity duration-75"
style={{ style={{
backgroundColor: avatar.skinColor || '#fcd3bf', backgroundColor: avatar.skinColor || '#fcd3bf',
opacity: trackingData.mouthOpen > 0.1 ? 1 : 0, opacity: trackingData.mouthOpen > 0.1 ? 1 : 0,
filter: 'blur(4px)', // Blends edges filter: 'blur(4px)',
borderRadius: '50%' borderRadius: '50%'
}} }}
/> />
{/* Mouth Sprite - Scales based on mouth openness */}
<Sprite <Sprite
imageSrc={processedImageUrl} imageSrc={processedImageUrl}
sourceRect={avatar.textureOpenMouth} sourceRect={avatar.textureOpenMouth}
className="w-full h-full" className="w-full h-full"
style={{ style={{
opacity: trackingData.mouthOpen > 0.05 ? 1 : 0, opacity: trackingData.mouthOpen > 0.05 ? 1 : 0,
// Scale open mouth based on volume
transform: `scaleY(${0.8 + trackingData.mouthOpen * 0.5})`, transform: `scaleY(${0.8 + trackingData.mouthOpen * 0.5})`,
}} }}
/> />
@ -280,7 +253,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
</div> </div>
)} )}
{/* Status Indicator overlay if tracking is lost */}
{(!cameraReady) && ( {(!cameraReady) && (
<div className="absolute inset-0 flex items-center justify-center bg-slate-900/80 z-20 rounded-xl backdrop-blur-sm"> <div className="absolute inset-0 flex items-center justify-center bg-slate-900/80 z-20 rounded-xl backdrop-blur-sm">
<div className="text-cyan-400 animate-pulse font-mono">INITIALIZING CAMERA LINK...</div> <div className="text-cyan-400 animate-pulse font-mono">INITIALIZING CAMERA LINK...</div>
@ -289,7 +261,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
</div> </div>
</div> </div>
{/* Control Deck */}
<div className="h-24 bg-slate-800 border-t border-slate-700 p-4 flex justify-center items-center gap-6 z-20"> <div className="h-24 bg-slate-800 border-t border-slate-700 p-4 flex justify-center items-center gap-6 z-20">
<div className="flex flex-col items-center"> <div className="flex flex-col items-center">
<span className="text-xs text-slate-400 mb-1 font-mono">MOUTH</span> <span className="text-xs text-slate-400 mb-1 font-mono">MOUTH</span>
@ -301,7 +272,6 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
<div className="flex flex-col items-center"> <div className="flex flex-col items-center">
<span className="text-xs text-slate-400 mb-1 font-mono">HEAD ROLL</span> <span className="text-xs text-slate-400 mb-1 font-mono">HEAD ROLL</span>
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden flex justify-center relative"> <div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden flex justify-center relative">
{/* Center marker */}
<div className="absolute w-[1px] h-full bg-slate-500 left-1/2"></div> <div className="absolute w-[1px] h-full bg-slate-500 left-1/2"></div>
<div <div
className="h-full bg-purple-500 transition-all duration-75 absolute" className="h-full bg-purple-500 transition-all duration-75 absolute"

26
hooks/useFaceTracking.ts → src/renderer/hooks/useFaceTracking.ts
View File

@ -1,6 +1,6 @@
import { useEffect, useRef, useState, useCallback } from 'react'; import { useEffect, useRef, useState, useCallback } from 'react';
import { FaceLandmarker, FilesetResolver, DrawingUtils } from '@mediapipe/tasks-vision'; import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
import { TrackingData } from '../types'; import { TrackingData } from '../../shared/types';
export const useFaceTracking = (videoElement: HTMLVideoElement | null) => { export const useFaceTracking = (videoElement: HTMLVideoElement | null) => {
const [isTracking, setIsTracking] = useState(false); const [isTracking, setIsTracking] = useState(false);
@ -19,16 +19,14 @@ export const useFaceTracking = (videoElement: HTMLVideoElement | null) => {
isBlinkingRight: false, isBlinkingRight: false,
}); });
// Initialize FaceLandmarker
useEffect(() => { useEffect(() => {
const initMediaPipe = async () => { const initMediaPipe = async () => {
try { try {
// Use specific version to match index.html import and prevent version mismatch
const filesetResolver = await FilesetResolver.forVisionTasks( const filesetResolver = await FilesetResolver.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm" "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm"
); );
faceLandmarkerRef.current = await FaceLandmarker.createFromOptions(filesetResolver, { faceLandmarkerRef.current = await FaceLandmarker.createFromOptions(filesetResolver, {
baseOptions: { baseOptions: {
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`, modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
delegate: "GPU" delegate: "GPU"
@ -40,8 +38,10 @@ export const useFaceTracking = (videoElement: HTMLVideoElement | null) => {
}); });
setIsLoading(false); setIsLoading(false);
if ((window as any).electronLog) (window as any).electronLog.info('MediaPipe faceLandmarker loaded');
} catch (error) { } catch (error) {
console.error("Failed to load MediaPipe:", error); console.error("Failed to load MediaPipe:", error);
if ((window as any).electronLog) (window as any).electronLog.error(`Failed to load MediaPipe: ${String(error)}`);
setIsLoading(false); setIsLoading(false);
} }
}; };
@ -56,7 +56,6 @@ export const useFaceTracking = (videoElement: HTMLVideoElement | null) => {
const predict = useCallback(() => { const predict = useCallback(() => {
if (!faceLandmarkerRef.current || !videoElement) return; if (!faceLandmarkerRef.current || !videoElement) return;
// Only predict if video is ready and playing
if (videoElement.readyState < 2) return; if (videoElement.readyState < 2) return;
const nowInMs = Date.now(); const nowInMs = Date.now();
@ -66,7 +65,6 @@ export const useFaceTracking = (videoElement: HTMLVideoElement | null) => {
const results = faceLandmarkerRef.current.detectForVideo(videoElement, nowInMs); const results = faceLandmarkerRef.current.detectForVideo(videoElement, nowInMs);
if (results.faceLandmarks && results.faceLandmarks.length > 0) { if (results.faceLandmarks && results.faceLandmarks.length > 0) {
// 1. Extract Blendshapes for Expression
const blendshapes = results.faceBlendshapes?.[0]?.categories; const blendshapes = results.faceBlendshapes?.[0]?.categories;
let mouthOpen = 0; let mouthOpen = 0;
@ -79,28 +77,20 @@ export const useFaceTracking = (videoElement: HTMLVideoElement | null) => {
eyeBlinkRight = blendshapes.find(c => c.categoryName === 'eyeBlinkRight')?.score || 0; eyeBlinkRight = blendshapes.find(c => c.categoryName === 'eyeBlinkRight')?.score || 0;
} }
// 2. Estimate Pose (simplified)
// MediaPipe gives a matrix, but often for 2D avatars, simple landmark delta is cleaner.
// We use specific landmarks to calculate roll, yaw, pitch approximation.
const landmarks = results.faceLandmarks[0]; const landmarks = results.faceLandmarks[0];
const leftEye = landmarks[33];
// Roll: Angle between eyes const rightEye = landmarks[263];
const leftEye = landmarks[33]; // Outer left eye
const rightEye = landmarks[263]; // Outer right eye
const dy = rightEye.y - leftEye.y; const dy = rightEye.y - leftEye.y;
const dx = rightEye.x - leftEye.x; const dx = rightEye.x - leftEye.x;
const roll = Math.atan2(dy, dx); const roll = Math.atan2(dy, dx);
// Yaw: Nose offset from center of eyes
const nose = landmarks[1]; const nose = landmarks[1];
const midPointX = (leftEye.x + rightEye.x) / 2; const midPointX = (leftEye.x + rightEye.x) / 2;
const yaw = (nose.x - midPointX) * 2; // sensitivity const yaw = (nose.x - midPointX) * 2;
// Pitch: Nose offset vertical
const midPointY = (leftEye.y + rightEye.y) / 2; const midPointY = (leftEye.y + rightEye.y) / 2;
const pitch = (nose.y - midPointY) * 2; const pitch = (nose.y - midPointY) * 2;
// Translation
const transX = (nose.x - 0.5) * 2; const transX = (nose.x - 0.5) * 2;
const transY = (nose.y - 0.5) * 2; const transY = (nose.y - 0.5) * 2;

23
src/renderer/index.tsx Normal file
View File

@ -0,0 +1,23 @@
import React from 'react';
import ReactDOM from 'react-dom/client';
import App from './App';
import './styles.css';
import ErrorBoundary from './components/ErrorBoundary';
const rootElement = document.getElementById('root');
if (!rootElement) {
throw new Error("Could not find root element to mount to");
}
const root = ReactDOM.createRoot(rootElement);
if ((window as any).electronLog) {
(window as any).electronLog.info('renderer started, mounting React root');
}
root.render(
<React.StrictMode>
<ErrorBoundary>
<App />
</ErrorBoundary>
</React.StrictMode>
);

89
src/renderer/services/geminiService.ts Normal file
View File

@ -0,0 +1,89 @@
import type { } from '@google/genai';
const placeholderGenerate = async (prompt: string) => {
const text = encodeURIComponent((prompt || '').substring(0, 80));
return `data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='1400' height='900'><rect width='100%' height='100%' fill='%230f172a' /><text x='50%' y='50%' fill='white' font-size='40' font-family='Inter' dominant-baseline='middle' text-anchor='middle'>Placeholder: ${text}</text></svg>`;
};
/**
* Generate an avatar image. Flow:
* 1) If running in Electron with the preload available, call main via window.electronAPI (passes localStorage key)
* 2) Else, attempt to dynamically import @google/genai and call it from the renderer using key in localStorage
* 3) Fallback to placeholder
*/
export const generateAvatarImage = async (description: string): Promise<string> => {
try {
// Prefer the Electron main process if available (safer, no browser CORS issues)
if (typeof window !== 'undefined' && (window as any).electronAPI && (window as any).electronAPI.generateAvatar) {
try {
const apiKey = localStorage.getItem('GEMINI_API_KEY') || undefined;
const res = await (window as any).electronAPI.generateAvatar(description, apiKey);
if (res && res.image) return res.image;
} catch (e) {
console.warn('[geminiService] electronAPI.generateAvatar failed, falling through to client attempt', e);
}
}
// Try running the GenAI client in the renderer (may fail due to environment/CORS)
try {
const apiKey = localStorage.getItem('GEMINI_API_KEY') || (window as any)?.process?.env?.GEMINI_API_KEY || (window as any)?.process?.env?.API_KEY;
if (!apiKey) {
console.warn('[geminiService] No API key available for client-side generation');
return await placeholderGenerate(description);
}
// Dynamic import so bundlers only include this if actually used
const mod = await import('@google/genai');
const { GoogleGenAI } = mod as any;
const ai = new GoogleGenAI({ apiKey });
const prompt = `
Create a VTuber character sheet with a flat 2D anime style.
LAYOUT:
1. MAIN CHARACTER (Left side, takes up 70% of width):
- Front-facing view, head and shoulders.
- Neutral expression, eyes open, mouth closed.
2. EXPRESSION ASSETS (Right side, vertical column):
- Top: The same character's face with EYES CLOSED (for blinking).
- Bottom: The same character's face with MOUTH OPEN (for talking).
Character Description: ${description}
Style: Vibrant, clean lines, solid white or green background for easy keying.
`;
const response = await ai.models.generateContent({
model: 'gemini-3-pro-image-preview',
contents: {
parts: [{ text: prompt }]
},
config: {
imageConfig: {
aspectRatio: '16:9',
imageSize: '1K'
}
}
});
const parts = response?.candidates?.[0]?.content?.parts || [];
for (const part of parts) {
if (part.inlineData && part.inlineData.data) {
return `data:image/png;base64,${part.inlineData.data}`;
}
}
console.warn('[geminiService] No image data found in client response');
return await placeholderGenerate(description);
} catch (clientErr) {
console.warn('[geminiService] Client-side GenAI attempt failed:', clientErr);
return await placeholderGenerate(description);
}
} catch (err) {
console.error('[geminiService] Unexpected error:', err);
return await placeholderGenerate(description);
}
};
export default { generateAvatarImage };

8
services/imageService.ts → src/renderer/services/imageService.ts
View File

@ -1,4 +1,4 @@
import { Rect } from '../types'; import { Rect } from '../../shared/types';
export const fileToDataUrl = (file: File): Promise<string> => { export const fileToDataUrl = (file: File): Promise<string> => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -24,16 +24,12 @@ export const stitchAssets = async (
blinkSrc?: string, blinkSrc?: string,
talkSrc?: string talkSrc?: string
): Promise<{ imageUrl: string; mainBody: Rect; textureClosedEye?: Rect; textureOpenMouth?: Rect }> => { ): Promise<{ imageUrl: string; mainBody: Rect; textureClosedEye?: Rect; textureOpenMouth?: Rect }> => {
// Load images
const baseImg = await loadImage(baseSrc); const baseImg = await loadImage(baseSrc);
const blinkImg = blinkSrc ? await loadImage(blinkSrc) : null; const blinkImg = blinkSrc ? await loadImage(blinkSrc) : null;
const talkImg = talkSrc ? await loadImage(talkSrc) : null; const talkImg = talkSrc ? await loadImage(talkSrc) : null;
// Layout: Base on Left. Sidebar on Right containing Blink (top) and Talk (bottom).
// Sidebar width = max(blink.width, talk.width)
const sidebarWidth = Math.max(blinkImg?.width || 0, talkImg?.width || 0); const sidebarWidth = Math.max(blinkImg?.width || 0, talkImg?.width || 0);
// If there are no variants, just return the base image as is
if (sidebarWidth === 0) { if (sidebarWidth === 0) {
return { return {
imageUrl: baseSrc, imageUrl: baseSrc,
@ -50,10 +46,8 @@ export const stitchAssets = async (
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
if (!ctx) throw new Error("Could not get canvas context"); if (!ctx) throw new Error("Could not get canvas context");
// Draw Base
ctx.drawImage(baseImg, 0, 0); ctx.drawImage(baseImg, 0, 0);
// Calculate normalized rects
const mainBody: Rect = { const mainBody: Rect = {
x: 0, x: 0,
y: 0, y: 0,

45
services/visionService.ts → src/renderer/services/visionService.ts
View File

@ -1,11 +1,9 @@
import { FaceLandmarker, FilesetResolver, ImageSegmenter } from '@mediapipe/tasks-vision'; import { FaceLandmarker, FilesetResolver, ImageSegmenter } from '@mediapipe/tasks-vision';
import { Rect } from '../types'; import { Rect } from '../../shared/types';
let faceLandmarker: FaceLandmarker | null = null; let faceLandmarker: FaceLandmarker | null = null;
let imageSegmenter: ImageSegmenter | null = null; let imageSegmenter: ImageSegmenter | null = null;
// Initialize the vision model for static image analysis
const initVision = async () => { const initVision = async () => {
if (faceLandmarker) return; if (faceLandmarker) return;
@ -14,10 +12,12 @@ const initVision = async () => {
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm" "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.18/wasm"
); );
// Use CPU delegate in Electron/dev for deterministic behavior. GPU can be flaky across
// platforms and in headless contexts; switch to GPU later if you need performance.
faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, { faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
baseOptions: { baseOptions: {
modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`, modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
delegate: "GPU" delegate: "CPU"
}, },
runningMode: "IMAGE", runningMode: "IMAGE",
numFaces: 1 numFaces: 1
@ -27,7 +27,6 @@ const initVision = async () => {
} }
}; };
// Initialize the segmenter for background removal
const initSegmenter = async () => { const initSegmenter = async () => {
if (imageSegmenter) return; if (imageSegmenter) return;
@ -60,12 +59,28 @@ export const analyzeAvatarImage = async (imageUrl: string): Promise<{ leftEye: R
img.crossOrigin = "anonymous"; img.crossOrigin = "anonymous";
img.onload = () => { img.onload = () => {
try { try {
const result = faceLandmarker!.detect(img); // If the generated image is small, upscale it to improve detection reliability
const minSide = Math.min(img.width, img.height);
let detectorInput: HTMLImageElement | HTMLCanvasElement = img;
if (minSide < 256) {
const scale = Math.ceil(256 / minSide);
const c = document.createElement('canvas');
c.width = img.width * scale;
c.height = img.height * scale;
const cctx = c.getContext('2d');
if (cctx) cctx.drawImage(img, 0, 0, c.width, c.height);
detectorInput = c;
console.log('[vision] upscaled image for detection to', c.width, 'x', c.height);
} else {
console.log('[vision] using image size', img.width, 'x', img.height);
}
const result = faceLandmarker!.detect(detectorInput as any);
console.log('[vision] detection result', result);
if (result.faceLandmarks && result.faceLandmarks.length > 0) { if (result.faceLandmarks && result.faceLandmarks.length > 0) {
const landmarks = result.faceLandmarks[0]; const landmarks = result.faceLandmarks[0];
// Helper to calculate bounding box from landmark indices
const getRect = (indices: number[]): Rect => { const getRect = (indices: number[]): Rect => {
let minX = 1, minY = 1, maxX = 0, maxY = 0; let minX = 1, minY = 1, maxX = 0, maxY = 0;
@ -80,7 +95,6 @@ export const analyzeAvatarImage = async (imageUrl: string): Promise<{ leftEye: R
const w = maxX - minX; const w = maxX - minX;
const h = maxY - minY; const h = maxY - minY;
// Expand slightly to cover the area comfortably
const paddingX = w * 0.1; const paddingX = w * 0.1;
const paddingY = h * 0.1; const paddingY = h * 0.1;
@ -92,7 +106,6 @@ export const analyzeAvatarImage = async (imageUrl: string): Promise<{ leftEye: R
}; };
}; };
// MediaPipe Mesh Indices
const leftEyeIndices = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246]; const leftEyeIndices = [33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161, 246];
const rightEyeIndices = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398]; const rightEyeIndices = [362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385, 384, 398];
const mouthIndices = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146]; const mouthIndices = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146];
@ -101,25 +114,22 @@ export const analyzeAvatarImage = async (imageUrl: string): Promise<{ leftEye: R
const rightRect = getRect(rightEyeIndices); const rightRect = getRect(rightEyeIndices);
const mouthRect = getRect(mouthIndices); const mouthRect = getRect(mouthIndices);
// Sample Skin Color
const canvas = document.createElement('canvas'); const canvas = document.createElement('canvas');
canvas.width = img.width; canvas.width = img.width;
canvas.height = img.height; canvas.height = img.height;
const ctx = canvas.getContext('2d'); const ctx = canvas.getContext('2d');
let color = '#fcd3bf'; // Default fallback let color = '#fcd3bf';
if (ctx) { if (ctx) {
ctx.drawImage(img, 0, 0); ctx.drawImage(img, 0, 0);
// Landmark 123 is on the left cheek bone area
const sampleIdx = 123; const sampleIdx = 123;
const lx = Math.floor(landmarks[sampleIdx].x * img.width); const lx = Math.floor(landmarks[sampleIdx].x * img.width);
const ly = Math.floor(landmarks[sampleIdx].y * img.height); const ly = Math.floor(landmarks[sampleIdx].y * img.height);
if (lx >= 0 && lx < img.width && ly >= 0 && ly < img.height) { if (lx >= 0 && lx < img.width && ly >= 0 && ly < img.height) {
const pixel = ctx.getImageData(lx, ly, 1, 1).data; const pixel = ctx.getImageData(lx, ly, 1, 1).data;
// Convert rgb to hex for input type="color"
const toHex = (c: number) => { const toHex = (c: number) => {
const hex = c.toString(16); const hex = c.toString(16);
return hex.length === 1 ? "0" + hex : hex; return hex.length === 1 ? "0" + hex : hex;
@ -161,7 +171,6 @@ export const removeBackground = async (imageUrl: string): Promise<string> => {
img.crossOrigin = "anonymous"; img.crossOrigin = "anonymous";
img.onload = () => { img.onload = () => {
try { try {
// 1. Segment the image
const segmentResult = imageSegmenter!.segment(img); const segmentResult = imageSegmenter!.segment(img);
const confidenceMasks = segmentResult.confidenceMasks; const confidenceMasks = segmentResult.confidenceMasks;
@ -170,7 +179,6 @@ export const removeBackground = async (imageUrl: string): Promise<string> => {
return; return;
} }
// 2. Create canvas and context
const canvas = document.createElement('canvas'); const canvas = document.createElement('canvas');
canvas.width = img.width; canvas.width = img.width;
canvas.height = img.height; canvas.height = img.height;
@ -181,24 +189,17 @@ export const removeBackground = async (imageUrl: string): Promise<string> => {
return; return;
} }
// 3. Draw original image
ctx.drawImage(img, 0, 0); ctx.drawImage(img, 0, 0);
const imageData = ctx.getImageData(0, 0, img.width, img.height); const imageData = ctx.getImageData(0, 0, img.width, img.height);
const pixels = imageData.data; const pixels = imageData.data;
// 4. Apply mask
// The selfie_segmenter output mask is a Float32Array where values
// indicate confidence of being a person (0.0 to 1.0).
const mask = confidenceMasks[0].getAsFloat32Array(); const mask = confidenceMasks[0].getAsFloat32Array();
for (let i = 0; i < mask.length; i++) { for (let i = 0; i < mask.length; i++) {
// Threshold for person confidence (0.3 is usually a good balance for hair details)
const confidence = mask[i]; const confidence = mask[i];
if (confidence < 0.3) { if (confidence < 0.3) {
pixels[i * 4 + 3] = 0; // Set Alpha to 0 pixels[i * 4 + 3] = 0; // Set Alpha to 0
} else { } else {
// Optional: Soft edges
// pixels[i * 4 + 3] = Math.floor(confidence * 255);
} }
} }

12
src/renderer/styles.css Normal file
View File

@ -0,0 +1,12 @@
@import "tailwindcss";
/* App-level custom styles */
body {
margin: 0;
background-color: #0f172a;
color: #f8fafc;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
#root { min-height: 100vh; }

61
src/shared/types.ts Normal file
View File

@ -0,0 +1,61 @@
export enum AppState {
SETUP = 'SETUP',
CREATION = 'CREATION',
RIGGING = 'RIGGING',
STUDIO = 'STUDIO',
}
export interface Rect {
x: number;
y: number;
w: number;
h: number;
}
export interface AvatarConfig {
imageUrl: string;
name: string;
description: string;
leftEye?: Rect;
rightEye?: Rect;
mouth?: Rect;
skinColor?: string;
textureClosedEye?: Rect;
textureOpenMouth?: Rect;
mainBody?: Rect;
chromaKeyColor?: string;
}
export interface TrackingData {
rotationX: number; // Pitch
rotationY: number; // Yaw
rotationZ: number; // Roll
translationX: number;
translationY: number;
mouthOpen: number;
isBlinkingLeft: boolean;
isBlinkingRight: boolean;
}
export interface AIStudio {
hasSelectedApiKey(): Promise<boolean>;
setApiKey(key: string): Promise<{ ok: boolean; error?: string; persisted?: boolean; warning?: string }>;
clearApiKey(): Promise<{ ok: boolean; error?: string; persisted?: boolean; warning?: string }>;
// Optional fields returned when persistence fails or succeeds
// setApiKey may return { ok:true, persisted:false, warning:'...'} or {ok:true, persisted:true}
// We'll model this loosely via the above return shape and consumers should check persisted/warning at runtime.
}
declare global {
interface Window {
aistudio?: AIStudio;
electronAPI?: {
generateAvatar: (prompt: string, apiKey?: string) => Promise<{ image: string }>
};
electronLog?: {
info: (m: string) => void;
warn: (m: string) => void;
error: (m: string) => void;
};
}
}

36
tailwind.config.cjs Normal file
View File

@ -0,0 +1,36 @@
module.exports = {
content: [
'./index.html',
'./src/**/*.{js,ts,jsx,tsx}',
'./components/**/*.{js,ts,jsx,tsx}',
'./**/*.{html,js,ts,jsx,tsx}'
],
theme: {
extend: {},
},
/*
Safelist: some classes are generated dynamically at runtime (template literals,
props-driven classNames, or libraries that build class strings). Tailwind's
static scanner can't see those, so include patterns here to ensure the
utilities are generated in dev/production builds.
*/
safelist: [
// background colors (with optional slash opacity like bg-cyan-500/10)
{ pattern: /bg-(?:cyan|purple|pink|green|slate|yellow|white|black)-(?:50|100|200|300|400|500|600|700|800|900)(?:\/\d{1,3})?/, variants: ['hover', 'focus', 'active', 'md', 'lg', 'peer-checked', 'group-hover'] },
// text colors
{ pattern: /text-(?:slate|cyan|green|pink|purple|yellow|white|black)-(?:50|100|200|300|400|500|600|700|800|900)/, variants: ['hover', 'focus', 'md', 'lg'] },
// z-index utilities used in code
{ pattern: /z-\d+/, variants: ['md', 'lg'] },
// width/height arbitrary values (w-[...], h-[...])
{ pattern: /w-\[.*\]/ },
{ pattern: /h-\[.*\]/ },
// fractional/utility sizes
{ pattern: /w-(?:full|auto|\[1px\]|\[600px\])/, variants: ['md'] },
{ pattern: /h-(?:full|screen|\[70vh\])/, variants: ['md'] },
// grid/cols
{ pattern: /grid-cols-\d+/ },
// opacity utilities
{ pattern: /bg-[\w-]+\/(?:\d{1,3})/, variants: ['hover', 'focus'] },
],
plugins: [],
};

2
tsconfig.json
View File

@ -20,7 +20,7 @@
"jsx": "react-jsx", "jsx": "react-jsx",
"paths": { "paths": {
"@/*": [ "@/*": [
"./*" "./src/*"
] ]
}, },
"allowImportingTsExtensions": true, "allowImportingTsExtensions": true,

11
types.ts
View File

@ -42,3 +42,14 @@ export interface AIStudio {
hasSelectedApiKey(): Promise<boolean>; hasSelectedApiKey(): Promise<boolean>;
openSelectKey(): Promise<void>; openSelectKey(): Promise<void>;
} }
// Make `window.aistudio` available globally in the browser environment
declare global {
interface Window {
aistudio?: AIStudio;
// electronAPI is injected by the preload script when running inside Electron
electronAPI?: {
generateAvatar: (prompt: string) => Promise<{ image: string }>
};
}
}

34
vite.config.ts
View File

@ -1,23 +1,25 @@
import path from 'path'; import path from 'path';
import { defineConfig, loadEnv } from 'vite'; import { defineConfig, loadEnv } from 'vite';
import react from '@vitejs/plugin-react'; import react from '@vitejs/plugin-react';
import tailwindcss from '@tailwindcss/vite'
export default defineConfig(({ mode }) => { export default defineConfig(({ mode }) => {
const env = loadEnv(mode, '.', ''); const env = loadEnv(mode, '.', '');
return { return {
server: { server: {
port: 3000, port: 5174,
host: '0.0.0.0', host: '0.0.0.0',
}, },
plugins: [react()], plugins: [react(), tailwindcss()],
define: { define: {
'process.env.API_KEY': JSON.stringify(env.GEMINI_API_KEY), 'process.env.API_KEY': JSON.stringify(env.GEMINI_API_KEY),
'process.env.GEMINI_API_KEY': JSON.stringify(env.GEMINI_API_KEY) 'process.env.GEMINI_API_KEY': JSON.stringify(env.GEMINI_API_KEY)
}, },
resolve: { resolve: {
alias: { alias: {
'@': path.resolve(__dirname, '.'), // Point @ to the src folder so imports can use @/renderer or @/shared
} '@': path.resolve(__dirname, 'src'),
} }
}; }
};
}); });