feat: Initialize Gemini V-Studio project setup

Sets up the foundational project structure, including:
- Vite for build tooling.
- React for the UI.
- Tailwind CSS for styling.
- MediaPipe for face tracking capabilities.
- Gemini API integration for avatar generation.
- Basic configuration files (package.json, vite.config.ts, tsconfig.json).
- Initial README with local run instructions.
- Core types and a basic Gemini service for image generation.
This commit is contained in:
James Twose
2025-11-20 20:45:25 +01:00
parent bbd1a19152
commit b6017794a5
17 changed files with 1279 additions and 8 deletions
+113
View File
@@ -0,0 +1,113 @@
import React, { useState } from 'react';
import { generateAvatarImage } from '../services/geminiService';
import { analyzeAvatarImage } from '../services/visionService';
import LoadingSpinner from './LoadingSpinner';
import { Rect } from '../types';
interface AvatarCreatorProps {
onAvatarGenerated: (url: string, name: string, initialData?: { leftEye: Rect, rightEye: Rect, mouth: Rect, skinColor: string }) => void;
}
const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
const [prompt, setPrompt] = useState('');
const [name, setName] = useState('');
const [status, setStatus] = useState<'idle' | 'generating' | 'analyzing'>('idle');
const [error, setError] = useState<string | null>(null);
const handleGenerate = async () => {
if (!prompt || !name) return;
setStatus('generating');
setError(null);
try {
// 1. Generate Image
const imageUrl = await generateAvatarImage(prompt);
// 2. Analyze Image for Landmarks (Initial guess)
setStatus('analyzing');
const analysisData = await analyzeAvatarImage(imageUrl);
// 3. Pass to parent (to go to Rigging)
if (analysisData) {
onAvatarGenerated(imageUrl, name, analysisData);
} else {
onAvatarGenerated(imageUrl, name);
}
} catch (err) {
console.error(err);
setError("Failed to generate avatar. Please try again.");
} finally {
setStatus('idle');
}
};
return (
<div className="max-w-2xl mx-auto bg-slate-800/50 backdrop-blur-lg border border-slate-700 p-8 rounded-2xl shadow-2xl">
<div className="text-center mb-8">
<h2 className="text-3xl font-bold text-transparent bg-clip-text bg-gradient-to-r from-cyan-400 to-purple-500 mb-2">
Design Your Avatar
</h2>
<p className="text-slate-400">
Describe your dream VTuber model and let Gemini bring it to life.
</p>
</div>
<div className="space-y-6">
<div>
<label className="block text-sm font-medium text-slate-300 mb-2">Model Name</label>
<input
type="text"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder="e.g., Neon Kitsune"
className="w-full bg-slate-900/50 border border-slate-600 rounded-xl px-4 py-3 text-white placeholder-slate-500 focus:ring-2 focus:ring-cyan-500 focus:border-transparent transition-all outline-none"
/>
</div>
<div>
<label className="block text-sm font-medium text-slate-300 mb-2">Description</label>
<textarea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
placeholder="e.g., A cyberpunk anime girl with neon blue hair, glowing headphones, wearing a futuristic jacket..."
className="w-full h-32 bg-slate-900/50 border border-slate-600 rounded-xl px-4 py-3 text-white placeholder-slate-500 focus:ring-2 focus:ring-cyan-500 focus:border-transparent transition-all outline-none resize-none"
/>
</div>
{error && (
<div className="p-3 bg-red-500/20 border border-red-500/50 rounded-lg text-red-200 text-sm">
{error}
</div>
)}
<button
onClick={handleGenerate}
disabled={status !== 'idle' || !prompt || !name}
className={`w-full py-4 rounded-xl font-bold text-lg transition-all duration-200 ${
status !== 'idle' || !prompt || !name
? 'bg-slate-700 text-slate-500 cursor-not-allowed'
: 'bg-gradient-to-r from-cyan-500 to-blue-600 hover:from-cyan-400 hover:to-blue-500 text-white shadow-lg shadow-cyan-500/25 transform hover:scale-[1.02]'
}`}
>
{status !== 'idle' ? (
<div className="flex items-center justify-center gap-3">
<LoadingSpinner />
<span>{status === 'generating' ? 'Dreaming up Avatar...' : 'Analyzing Features...'}</span>
</div>
) : (
<div className="flex items-center justify-center gap-2">
<span>Generate Model</span>
<svg xmlns="http://www.w3.org/2000/svg" className="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-8.707l-3-3a1 1 0 00-1.414 1.414L10.586 9H7a1 1 0 100 2h3.586l-1.293 1.293a1 1 0 101.414 1.414l3-3a1 1 0 000-1.414z" clipRule="evenodd" />
</svg>
</div>
)}
</button>
</div>
</div>
);
};
export default AvatarCreator;
+11
View File
@@ -0,0 +1,11 @@
import React from 'react';
const LoadingSpinner: React.FC = () => (
<div className="flex justify-center items-center space-x-2">
<div className="w-4 h-4 bg-cyan-500 rounded-full animate-bounce" style={{ animationDelay: '0s' }}></div>
<div className="w-4 h-4 bg-purple-500 rounded-full animate-bounce" style={{ animationDelay: '0.1s' }}></div>
<div className="w-4 h-4 bg-pink-500 rounded-full animate-bounce" style={{ animationDelay: '0.2s' }}></div>
</div>
);
export default LoadingSpinner;
+224
View File
@@ -0,0 +1,224 @@
import React, { useState, useRef, useEffect } from 'react';
import { Rect } from '../types';
interface RiggingEditorProps {
imageUrl: string;
initialData?: { leftEye: Rect; rightEye: Rect; mouth: Rect; skinColor: string };
onComplete: (data: { leftEye: Rect; rightEye: Rect; mouth: Rect; skinColor: string }) => void;
}
type ActiveFeature = 'leftEye' | 'rightEye' | 'mouth' | null;
const ResizableBox: React.FC<{
rect: Rect;
color: string;
label: string;
isActive: boolean;
onUpdate: (rect: Rect) => void;
onActivate: () => void;
}> = ({ rect, color, label, isActive, onUpdate, onActivate }) => {
const boxRef = useRef<HTMLDivElement>(null);
const [isDragging, setIsDragging] = useState(false);
const [isResizing, setIsResizing] = useState(false);
const startPos = useRef({ x: 0, y: 0 });
const startRect = useRef<Rect>({ x: 0, y: 0, w: 0, h: 0 });
const handleMouseDown = (e: React.MouseEvent) => {
e.preventDefault();
e.stopPropagation();
onActivate();
setIsDragging(true);
startPos.current = { x: e.clientX, y: e.clientY };
startRect.current = { ...rect };
};
const handleResizeDown = (e: React.MouseEvent) => {
e.preventDefault();
e.stopPropagation();
onActivate();
setIsResizing(true);
startPos.current = { x: e.clientX, y: e.clientY };
startRect.current = { ...rect };
};
useEffect(() => {
const handleMouseMove = (e: MouseEvent) => {
if (!isDragging && !isResizing) return;
const parent = boxRef.current?.parentElement;
if (!parent) return;
const parentRect = parent.getBoundingClientRect();
const deltaX = (e.clientX - startPos.current.x) / parentRect.width;
const deltaY = (e.clientY - startPos.current.y) / parentRect.height;
if (isDragging) {
onUpdate({
...rect,
x: startRect.current.x + deltaX,
y: startRect.current.y + deltaY,
});
} else if (isResizing) {
onUpdate({
...rect,
w: Math.max(0.01, startRect.current.w + deltaX),
h: Math.max(0.01, startRect.current.h + deltaY),
});
}
};
const handleMouseUp = () => {
setIsDragging(false);
setIsResizing(false);
};
if (isDragging || isResizing) {
window.addEventListener('mousemove', handleMouseMove);
window.addEventListener('mouseup', handleMouseUp);
}
return () => {
window.removeEventListener('mousemove', handleMouseMove);
window.removeEventListener('mouseup', handleMouseUp);
};
}, [isDragging, isResizing, rect, onUpdate]);
return (
<div
ref={boxRef}
onMouseDown={handleMouseDown}
className={`absolute border-2 cursor-move group transition-colors ${isActive ? 'z-20' : 'z-10'}`}
style={{
left: `${rect.x * 100}%`,
top: `${rect.y * 100}%`,
width: `${rect.w * 100}%`,
height: `${rect.h * 100}%`,
borderColor: color,
backgroundColor: isActive ? `${color}20` : 'transparent',
}}
>
{/* Label */}
<div
className="absolute -top-6 left-0 text-xs font-bold px-1 rounded text-white whitespace-nowrap"
style={{ backgroundColor: color }}
>
{label}
</div>
{/* Resize Handle */}
<div
onMouseDown={handleResizeDown}
className="absolute bottom-0 right-0 w-4 h-4 bg-white border-2 cursor-nwse-resize opacity-0 group-hover:opacity-100 transition-opacity"
style={{ borderColor: color }}
/>
</div>
);
};
const RiggingEditor: React.FC<RiggingEditorProps> = ({ imageUrl, initialData, onComplete }) => {
const [leftEye, setLeftEye] = useState<Rect>(initialData?.leftEye || { x: 0.35, y: 0.4, w: 0.12, h: 0.08 });
const [rightEye, setRightEye] = useState<Rect>(initialData?.rightEye || { x: 0.53, y: 0.4, w: 0.12, h: 0.08 });
const [mouth, setMouth] = useState<Rect>(initialData?.mouth || { x: 0.45, y: 0.6, w: 0.1, h: 0.05 });
const [skinColor, setSkinColor] = useState<string>(initialData?.skinColor || '#fcd3bf');
const [activeFeature, setActiveFeature] = useState<ActiveFeature>(null);
return (
<div className="flex flex-col items-center h-full max-w-4xl mx-auto p-4">
<div className="text-center mb-6">
<h2 className="text-2xl font-bold text-white mb-2">Rig Your Avatar</h2>
<p className="text-slate-400">
Drag and resize the boxes to match your avatar's features.
This ensures the eyes blink correctly.
</p>
</div>
<div className="flex gap-8 w-full items-start">
{/* Editor Area */}
<div className="flex-1 bg-slate-800 p-4 rounded-xl border border-slate-700 flex justify-center">
<div className="relative inline-block select-none" style={{ width: '500px', maxWidth: '100%' }}>
<img
src={imageUrl}
alt="Rigging Target"
className="w-full h-auto rounded-lg pointer-events-none select-none block"
draggable={false}
/>
<ResizableBox
rect={leftEye}
color="#ef4444" // Red
label="Left Eye"
isActive={activeFeature === 'leftEye'}
onUpdate={setLeftEye}
onActivate={() => setActiveFeature('leftEye')}
/>
<ResizableBox
rect={rightEye}
color="#3b82f6" // Blue
label="Right Eye"
isActive={activeFeature === 'rightEye'}
onUpdate={setRightEye}
onActivate={() => setActiveFeature('rightEye')}
/>
<ResizableBox
rect={mouth}
color="#22c55e" // Green
label="Mouth"
isActive={activeFeature === 'mouth'}
onUpdate={setMouth}
onActivate={() => setActiveFeature('mouth')}
/>
</div>
</div>
{/* Sidebar Controls */}
<div className="w-64 flex flex-col gap-6 bg-slate-800/50 p-6 rounded-xl border border-slate-700 h-full">
<div>
<label className="block text-sm font-medium text-slate-300 mb-2">Eyelid Color</label>
<div className="flex items-center gap-3">
<input
type="color"
value={skinColor}
onChange={(e) => setSkinColor(e.target.value)}
className="w-10 h-10 rounded cursor-pointer border-0 p-0"
/>
<span className="text-xs text-slate-400 font-mono">{skinColor}</span>
</div>
<p className="text-xs text-slate-500 mt-2">
Pick the color of the skin above the eyes for realistic blinking.
</p>
</div>
<div className="space-y-2">
<div className="flex items-center gap-2 text-sm text-slate-300">
<div className="w-3 h-3 bg-red-500 rounded-full"></div>
<span>Left Eye Box</span>
</div>
<div className="flex items-center gap-2 text-sm text-slate-300">
<div className="w-3 h-3 bg-blue-500 rounded-full"></div>
<span>Right Eye Box</span>
</div>
<div className="flex items-center gap-2 text-sm text-slate-300">
<div className="w-3 h-3 bg-green-500 rounded-full"></div>
<span>Mouth Box</span>
</div>
</div>
<div className="mt-auto pt-6">
<button
onClick={() => onComplete({ leftEye, rightEye, mouth, skinColor })}
className="w-full py-3 bg-gradient-to-r from-cyan-500 to-blue-600 hover:from-cyan-400 hover:to-blue-500 text-white rounded-xl font-bold shadow-lg shadow-cyan-500/25 transform hover:scale-[1.02] transition-all"
>
Finish Rigging
</button>
</div>
</div>
</div>
</div>
);
};
export default RiggingEditor;
+259
View File
@@ -0,0 +1,259 @@
import React, { useEffect, useRef, useState } from 'react';
import { useFaceTracking } from '../hooks/useFaceTracking';
import { AvatarConfig } from '../types';
interface StudioProps {
avatar: AvatarConfig;
onBack: () => void;
}
const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
const videoRef = useRef<HTMLVideoElement>(null);
const [cameraReady, setCameraReady] = useState(false);
// We use the custom hook to get tracking data
const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current);
// Initialize Camera
useEffect(() => {
const startCamera = async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({
video: { width: 640, height: 480 }, // Lower res is fine for tracking
audio: false
});
if (videoRef.current) {
videoRef.current.srcObject = stream;
videoRef.current.onloadeddata = () => {
setCameraReady(true);
};
}
} catch (err) {
console.error("Error accessing camera:", err);
alert("Could not access camera. Please ensure permissions are granted.");
}
};
startCamera();
return () => {
// Cleanup stream
if (videoRef.current && videoRef.current.srcObject) {
const stream = videoRef.current.srcObject as MediaStream;
stream.getTracks().forEach(track => track.stop());
}
};
}, []);
// Start tracking when both camera and model are ready
useEffect(() => {
if (cameraReady && !isModelLoading) {
startTracking();
}
}, [cameraReady, isModelLoading, startTracking]);
// Calculate styles based on tracking data
const getAvatarStyle = () => {
// Deadzone for jitter reduction
const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val;
const rX = smooth(trackingData.rotationX); // Pitch
const rY = smooth(trackingData.rotationY); // Yaw
const rZ = smooth(trackingData.rotationZ); // Roll
const tX = smooth(trackingData.translationX);
const tY = smooth(trackingData.translationY);
// Bounce effect on mouth open (Speaking emulation)
const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0;
return {
transform: `
translate(${tX * 150}px, ${tY * 100 + bounce}px)
rotate(${rZ * 1}rad)
perspective(500px)
rotateX(${rX * 15}deg)
rotateY(${rY * -25}deg)
scale(${1 + trackingData.mouthOpen * 0.02})
`,
filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`, // Slight flash when speaking
transition: 'transform 0.1s ease-out, filter 0.1s ease'
};
};
return (
<div className="h-screen w-full flex flex-col bg-slate-900 overflow-hidden relative">
{/* Hidden Video Element for Tracking */}
<video
ref={videoRef}
autoPlay
playsInline
muted
className="absolute opacity-0 pointer-events-none w-1 h-1"
/>
{/* Top Bar */}
<div className="absolute top-0 left-0 right-0 z-20 p-4 flex justify-between items-center bg-gradient-to-b from-slate-900 to-transparent">
<button
onClick={onBack}
className="px-4 py-2 bg-slate-800/80 hover:bg-slate-700 backdrop-blur rounded-lg text-white font-medium transition-colors border border-slate-600"
>
Exit Studio
</button>
<div className="flex gap-2">
<div className={`px-3 py-1 rounded-full text-xs font-bold flex items-center gap-2 ${isModelLoading ? 'bg-yellow-500/20 text-yellow-400' : 'bg-green-500/20 text-green-400'}`}>
<span className={`w-2 h-2 rounded-full ${isModelLoading ? 'bg-yellow-400 animate-pulse' : 'bg-green-400'}`}></span>
{isModelLoading ? 'Loading Vision Model...' : 'Tracking Active'}
</div>
<div className="px-3 py-1 rounded-full text-xs font-bold bg-purple-500/20 text-purple-400 border border-purple-500/30">
{avatar.name}
</div>
</div>
</div>
{/* Main Stage */}
<div className="flex-1 relative flex items-center justify-center overflow-hidden">
{/* Background Grid/Effect */}
<div className="absolute inset-0 opacity-20"
style={{
backgroundImage: 'radial-gradient(#4f46e5 1px, transparent 1px)',
backgroundSize: '30px 30px'
}}>
</div>
<div className="absolute inset-0 bg-gradient-to-t from-slate-900 via-transparent to-slate-900 pointer-events-none"></div>
{/* Avatar Container */}
<div className="relative w-[600px] h-[600px] flex items-center justify-center z-10">
<div
className="relative w-full h-full flex items-center justify-center"
style={getAvatarStyle()}
>
<img
src={avatar.imageUrl}
alt="Avatar"
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
/>
{/* Dynamic Eyelids */}
{avatar.leftEye && avatar.skinColor && (
<div
className="absolute pointer-events-none"
style={{
left: `${avatar.leftEye.x * 100}%`,
top: `${avatar.leftEye.y * 100}%`,
width: `${avatar.leftEye.w * 100}%`,
height: `${avatar.leftEye.h * 100}%`,
backgroundColor: avatar.skinColor,
transform: `scaleY(${trackingData.isBlinkingLeft ? 1 : 0})`,
transformOrigin: 'top',
transition: 'transform 0.1s cubic-bezier(0.4, 0, 0.2, 1)', // Snappy blink
borderRadius: '0 0 40% 40%'
}}
/>
)}
{avatar.rightEye && avatar.skinColor && (
<div
className="absolute pointer-events-none"
style={{
left: `${avatar.rightEye.x * 100}%`,
top: `${avatar.rightEye.y * 100}%`,
width: `${avatar.rightEye.w * 100}%`,
height: `${avatar.rightEye.h * 100}%`,
backgroundColor: avatar.skinColor,
transform: `scaleY(${trackingData.isBlinkingRight ? 1 : 0})`,
transformOrigin: 'top',
transition: 'transform 0.1s cubic-bezier(0.4, 0, 0.2, 1)', // Snappy blink
borderRadius: '0 0 40% 40%'
}}
/>
)}
{/* Dynamic Mouth Animation */}
{avatar.mouth && (
<div
className="absolute pointer-events-none flex items-center justify-center z-10"
style={{
left: `${avatar.mouth.x * 100}%`,
top: `${avatar.mouth.y * 100}%`,
width: `${avatar.mouth.w * 100}%`,
height: `${avatar.mouth.h * 100}%`,
}}
>
{/* Skin Patch - Hides the static closed mouth when speaking */}
<div
className="absolute w-[120%] h-[120%] transition-opacity duration-75"
style={{
backgroundColor: avatar.skinColor || '#fcd3bf',
opacity: trackingData.mouthOpen > 0.1 ? 1 : 0,
filter: 'blur(3px)', // Blends edges
borderRadius: '40%'
}}
/>
{/* Mouth Interior - Scales based on mouth openness */}
<div
className="relative w-full h-full bg-[#4a1212] border-2 border-[#2d0a0a] overflow-hidden origin-center transition-transform duration-75"
style={{
borderRadius: '50% 50% 50% 50% / 50% 50% 30% 30%', // Slightly more jaw-like shape
// trackingData.mouthOpen is 0-1. We amplify it for better visuals.
transform: `scaleY(${Math.min(1.2, trackingData.mouthOpen * 4)}) scaleX(${0.9 + trackingData.mouthOpen * 0.1})`,
opacity: trackingData.mouthOpen > 0.05 ? 1 : 0,
}}
>
{/* Tongue */}
<div
className="absolute bottom-[-20%] left-1/2 -translate-x-1/2 w-[80%] h-[60%] bg-[#d45d5d] rounded-t-full"
/>
</div>
</div>
)}
</div>
{/* Optional: Status Indicator overlay if tracking is lost (all 0s usually) or just visual flair */}
{(!cameraReady) && (
<div className="absolute inset-0 flex items-center justify-center bg-slate-900/80 z-20 rounded-xl backdrop-blur-sm">
<div className="text-cyan-400 animate-pulse font-mono">INITIALIZING CAMERA LINK...</div>
</div>
)}
</div>
</div>
{/* Control Deck */}
<div className="h-24 bg-slate-800 border-t border-slate-700 p-4 flex justify-center items-center gap-6 z-20">
<div className="flex flex-col items-center">
<span className="text-xs text-slate-400 mb-1 font-mono">MOUTH</span>
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden">
<div className="h-full bg-cyan-400 transition-all duration-75" style={{ width: `${Math.min(trackingData.mouthOpen * 100, 100)}%` }}></div>
</div>
</div>
<div className="flex flex-col items-center">
<span className="text-xs text-slate-400 mb-1 font-mono">HEAD ROLL</span>
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden flex justify-center relative">
{/* Center marker */}
<div className="absolute w-[1px] h-full bg-slate-500 left-1/2"></div>
<div
className="h-full bg-purple-500 transition-all duration-75 absolute"
style={{
width: `${Math.abs(trackingData.rotationZ * 50)}%`,
left: trackingData.rotationZ < 0 ? 'auto' : '50%',
right: trackingData.rotationZ < 0 ? '50%' : 'auto'
}}
></div>
</div>
</div>
<div className="flex flex-col items-center">
<span className="text-xs text-slate-400 mb-1 font-mono">BLINK</span>
<div className="flex gap-2">
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingLeft ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingRight ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
</div>
</div>
</div>
</div>
);
};
export default Studio;