b6017794a5
Sets up the foundational project structure, including: - Vite for build tooling. - React for the UI. - Tailwind CSS for styling. - MediaPipe for face tracking capabilities. - Gemini API integration for avatar generation. - Basic configuration files (package.json, vite.config.ts, tsconfig.json). - Initial README with local run instructions. - Core types and a basic Gemini service for image generation.
259 lines
11 KiB
TypeScript
259 lines
11 KiB
TypeScript
import React, { useEffect, useRef, useState } from 'react';
|
|
import { useFaceTracking } from '../hooks/useFaceTracking';
|
|
import { AvatarConfig } from '../types';
|
|
|
|
interface StudioProps {
|
|
avatar: AvatarConfig;
|
|
onBack: () => void;
|
|
}
|
|
|
|
const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
|
|
const videoRef = useRef<HTMLVideoElement>(null);
|
|
const [cameraReady, setCameraReady] = useState(false);
|
|
|
|
// We use the custom hook to get tracking data
|
|
const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current);
|
|
|
|
// Initialize Camera
|
|
useEffect(() => {
|
|
const startCamera = async () => {
|
|
try {
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
video: { width: 640, height: 480 }, // Lower res is fine for tracking
|
|
audio: false
|
|
});
|
|
if (videoRef.current) {
|
|
videoRef.current.srcObject = stream;
|
|
videoRef.current.onloadeddata = () => {
|
|
setCameraReady(true);
|
|
};
|
|
}
|
|
} catch (err) {
|
|
console.error("Error accessing camera:", err);
|
|
alert("Could not access camera. Please ensure permissions are granted.");
|
|
}
|
|
};
|
|
|
|
startCamera();
|
|
|
|
return () => {
|
|
// Cleanup stream
|
|
if (videoRef.current && videoRef.current.srcObject) {
|
|
const stream = videoRef.current.srcObject as MediaStream;
|
|
stream.getTracks().forEach(track => track.stop());
|
|
}
|
|
};
|
|
}, []);
|
|
|
|
// Start tracking when both camera and model are ready
|
|
useEffect(() => {
|
|
if (cameraReady && !isModelLoading) {
|
|
startTracking();
|
|
}
|
|
}, [cameraReady, isModelLoading, startTracking]);
|
|
|
|
// Calculate styles based on tracking data
|
|
const getAvatarStyle = () => {
|
|
// Deadzone for jitter reduction
|
|
const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val;
|
|
|
|
const rX = smooth(trackingData.rotationX); // Pitch
|
|
const rY = smooth(trackingData.rotationY); // Yaw
|
|
const rZ = smooth(trackingData.rotationZ); // Roll
|
|
const tX = smooth(trackingData.translationX);
|
|
const tY = smooth(trackingData.translationY);
|
|
|
|
// Bounce effect on mouth open (Speaking emulation)
|
|
const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0;
|
|
|
|
return {
|
|
transform: `
|
|
translate(${tX * 150}px, ${tY * 100 + bounce}px)
|
|
rotate(${rZ * 1}rad)
|
|
perspective(500px)
|
|
rotateX(${rX * 15}deg)
|
|
rotateY(${rY * -25}deg)
|
|
scale(${1 + trackingData.mouthOpen * 0.02})
|
|
`,
|
|
filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`, // Slight flash when speaking
|
|
transition: 'transform 0.1s ease-out, filter 0.1s ease'
|
|
};
|
|
};
|
|
|
|
return (
|
|
<div className="h-screen w-full flex flex-col bg-slate-900 overflow-hidden relative">
|
|
{/* Hidden Video Element for Tracking */}
|
|
<video
|
|
ref={videoRef}
|
|
autoPlay
|
|
playsInline
|
|
muted
|
|
className="absolute opacity-0 pointer-events-none w-1 h-1"
|
|
/>
|
|
|
|
{/* Top Bar */}
|
|
<div className="absolute top-0 left-0 right-0 z-20 p-4 flex justify-between items-center bg-gradient-to-b from-slate-900 to-transparent">
|
|
<button
|
|
onClick={onBack}
|
|
className="px-4 py-2 bg-slate-800/80 hover:bg-slate-700 backdrop-blur rounded-lg text-white font-medium transition-colors border border-slate-600"
|
|
>
|
|
← Exit Studio
|
|
</button>
|
|
<div className="flex gap-2">
|
|
<div className={`px-3 py-1 rounded-full text-xs font-bold flex items-center gap-2 ${isModelLoading ? 'bg-yellow-500/20 text-yellow-400' : 'bg-green-500/20 text-green-400'}`}>
|
|
<span className={`w-2 h-2 rounded-full ${isModelLoading ? 'bg-yellow-400 animate-pulse' : 'bg-green-400'}`}></span>
|
|
{isModelLoading ? 'Loading Vision Model...' : 'Tracking Active'}
|
|
</div>
|
|
<div className="px-3 py-1 rounded-full text-xs font-bold bg-purple-500/20 text-purple-400 border border-purple-500/30">
|
|
{avatar.name}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Main Stage */}
|
|
<div className="flex-1 relative flex items-center justify-center overflow-hidden">
|
|
{/* Background Grid/Effect */}
|
|
<div className="absolute inset-0 opacity-20"
|
|
style={{
|
|
backgroundImage: 'radial-gradient(#4f46e5 1px, transparent 1px)',
|
|
backgroundSize: '30px 30px'
|
|
}}>
|
|
</div>
|
|
|
|
<div className="absolute inset-0 bg-gradient-to-t from-slate-900 via-transparent to-slate-900 pointer-events-none"></div>
|
|
|
|
{/* Avatar Container */}
|
|
<div className="relative w-[600px] h-[600px] flex items-center justify-center z-10">
|
|
<div
|
|
className="relative w-full h-full flex items-center justify-center"
|
|
style={getAvatarStyle()}
|
|
>
|
|
<img
|
|
src={avatar.imageUrl}
|
|
alt="Avatar"
|
|
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
|
|
/>
|
|
|
|
{/* Dynamic Eyelids */}
|
|
{avatar.leftEye && avatar.skinColor && (
|
|
<div
|
|
className="absolute pointer-events-none"
|
|
style={{
|
|
left: `${avatar.leftEye.x * 100}%`,
|
|
top: `${avatar.leftEye.y * 100}%`,
|
|
width: `${avatar.leftEye.w * 100}%`,
|
|
height: `${avatar.leftEye.h * 100}%`,
|
|
backgroundColor: avatar.skinColor,
|
|
transform: `scaleY(${trackingData.isBlinkingLeft ? 1 : 0})`,
|
|
transformOrigin: 'top',
|
|
transition: 'transform 0.1s cubic-bezier(0.4, 0, 0.2, 1)', // Snappy blink
|
|
borderRadius: '0 0 40% 40%'
|
|
}}
|
|
/>
|
|
)}
|
|
|
|
{avatar.rightEye && avatar.skinColor && (
|
|
<div
|
|
className="absolute pointer-events-none"
|
|
style={{
|
|
left: `${avatar.rightEye.x * 100}%`,
|
|
top: `${avatar.rightEye.y * 100}%`,
|
|
width: `${avatar.rightEye.w * 100}%`,
|
|
height: `${avatar.rightEye.h * 100}%`,
|
|
backgroundColor: avatar.skinColor,
|
|
transform: `scaleY(${trackingData.isBlinkingRight ? 1 : 0})`,
|
|
transformOrigin: 'top',
|
|
transition: 'transform 0.1s cubic-bezier(0.4, 0, 0.2, 1)', // Snappy blink
|
|
borderRadius: '0 0 40% 40%'
|
|
}}
|
|
/>
|
|
)}
|
|
|
|
{/* Dynamic Mouth Animation */}
|
|
{avatar.mouth && (
|
|
<div
|
|
className="absolute pointer-events-none flex items-center justify-center z-10"
|
|
style={{
|
|
left: `${avatar.mouth.x * 100}%`,
|
|
top: `${avatar.mouth.y * 100}%`,
|
|
width: `${avatar.mouth.w * 100}%`,
|
|
height: `${avatar.mouth.h * 100}%`,
|
|
}}
|
|
>
|
|
{/* Skin Patch - Hides the static closed mouth when speaking */}
|
|
<div
|
|
className="absolute w-[120%] h-[120%] transition-opacity duration-75"
|
|
style={{
|
|
backgroundColor: avatar.skinColor || '#fcd3bf',
|
|
opacity: trackingData.mouthOpen > 0.1 ? 1 : 0,
|
|
filter: 'blur(3px)', // Blends edges
|
|
borderRadius: '40%'
|
|
}}
|
|
/>
|
|
|
|
{/* Mouth Interior - Scales based on mouth openness */}
|
|
<div
|
|
className="relative w-full h-full bg-[#4a1212] border-2 border-[#2d0a0a] overflow-hidden origin-center transition-transform duration-75"
|
|
style={{
|
|
borderRadius: '50% 50% 50% 50% / 50% 50% 30% 30%', // Slightly more jaw-like shape
|
|
// trackingData.mouthOpen is 0-1. We amplify it for better visuals.
|
|
transform: `scaleY(${Math.min(1.2, trackingData.mouthOpen * 4)}) scaleX(${0.9 + trackingData.mouthOpen * 0.1})`,
|
|
opacity: trackingData.mouthOpen > 0.05 ? 1 : 0,
|
|
}}
|
|
>
|
|
{/* Tongue */}
|
|
<div
|
|
className="absolute bottom-[-20%] left-1/2 -translate-x-1/2 w-[80%] h-[60%] bg-[#d45d5d] rounded-t-full"
|
|
/>
|
|
</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
|
|
{/* Optional: Status Indicator overlay if tracking is lost (all 0s usually) or just visual flair */}
|
|
{(!cameraReady) && (
|
|
<div className="absolute inset-0 flex items-center justify-center bg-slate-900/80 z-20 rounded-xl backdrop-blur-sm">
|
|
<div className="text-cyan-400 animate-pulse font-mono">INITIALIZING CAMERA LINK...</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
{/* Control Deck */}
|
|
<div className="h-24 bg-slate-800 border-t border-slate-700 p-4 flex justify-center items-center gap-6 z-20">
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">MOUTH</span>
|
|
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden">
|
|
<div className="h-full bg-cyan-400 transition-all duration-75" style={{ width: `${Math.min(trackingData.mouthOpen * 100, 100)}%` }}></div>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">HEAD ROLL</span>
|
|
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden flex justify-center relative">
|
|
{/* Center marker */}
|
|
<div className="absolute w-[1px] h-full bg-slate-500 left-1/2"></div>
|
|
<div
|
|
className="h-full bg-purple-500 transition-all duration-75 absolute"
|
|
style={{
|
|
width: `${Math.abs(trackingData.rotationZ * 50)}%`,
|
|
left: trackingData.rotationZ < 0 ? 'auto' : '50%',
|
|
right: trackingData.rotationZ < 0 ? '50%' : 'auto'
|
|
}}
|
|
></div>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">BLINK</span>
|
|
<div className="flex gap-2">
|
|
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingLeft ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
|
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingRight ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
);
|
|
};
|
|
|
|
export default Studio; |