3eff403fb4
The Gemini service has been updated to generate a character sheet rather than a single avatar image. This sheet includes the main character and separate assets for closed eyes and an open mouth. The `AvatarConfig` type and `RiggingEditor` component have been extended to handle these new expression assets (`textureClosedEye`, `textureOpenMouth`). A new `Sprite` component has been added to `Studio.tsx` to correctly render these specific regions from the generated character sheet. The UI has been updated to reflect the new generation process.
299 lines
12 KiB
TypeScript
299 lines
12 KiB
TypeScript
|
|
import React, { useEffect, useRef, useState } from 'react';
|
|
import { useFaceTracking } from '../hooks/useFaceTracking';
|
|
import { AvatarConfig, Rect } from '../types';
|
|
|
|
interface StudioProps {
|
|
avatar: AvatarConfig;
|
|
onBack: () => void;
|
|
}
|
|
|
|
/**
|
|
* Sprite Component
|
|
* Renders a specific crop of the source image into a target container.
|
|
*/
|
|
const Sprite: React.FC<{
|
|
imageSrc: string;
|
|
sourceRect: Rect;
|
|
style?: React.CSSProperties;
|
|
className?: string;
|
|
}> = ({ imageSrc, sourceRect, style, className }) => {
|
|
// To display a cropped region (sourceRect) of the image, we use an inner <img>
|
|
// positioned negatively and scaled up.
|
|
// Example: If sourceRect.w is 0.1 (10%), the image must be scaled to 10x (1000%) size.
|
|
const widthScale = 100 / (sourceRect.w * 100);
|
|
const heightScale = 100 / (sourceRect.h * 100);
|
|
|
|
return (
|
|
<div
|
|
className={`overflow-hidden relative ${className}`}
|
|
style={style}
|
|
>
|
|
<img
|
|
src={imageSrc}
|
|
alt=""
|
|
style={{
|
|
position: 'absolute',
|
|
top: `-${sourceRect.y * 100 * heightScale}%`,
|
|
left: `-${sourceRect.x * 100 * widthScale}%`,
|
|
width: `${widthScale * 100}%`,
|
|
height: `${heightScale * 100}%`,
|
|
maxWidth: 'none',
|
|
maxHeight: 'none',
|
|
pointerEvents: 'none'
|
|
}}
|
|
/>
|
|
</div>
|
|
);
|
|
};
|
|
|
|
const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
|
|
const videoRef = useRef<HTMLVideoElement>(null);
|
|
const [cameraReady, setCameraReady] = useState(false);
|
|
|
|
// We use the custom hook to get tracking data
|
|
const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current);
|
|
|
|
// Initialize Camera
|
|
useEffect(() => {
|
|
const startCamera = async () => {
|
|
try {
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
video: { width: 640, height: 480 }, // Lower res is fine for tracking
|
|
audio: false
|
|
});
|
|
if (videoRef.current) {
|
|
videoRef.current.srcObject = stream;
|
|
videoRef.current.onloadeddata = () => {
|
|
setCameraReady(true);
|
|
};
|
|
}
|
|
} catch (err) {
|
|
console.error("Error accessing camera:", err);
|
|
alert("Could not access camera. Please ensure permissions are granted.");
|
|
}
|
|
};
|
|
|
|
startCamera();
|
|
|
|
return () => {
|
|
// Cleanup stream
|
|
if (videoRef.current && videoRef.current.srcObject) {
|
|
const stream = videoRef.current.srcObject as MediaStream;
|
|
stream.getTracks().forEach(track => track.stop());
|
|
}
|
|
};
|
|
}, []);
|
|
|
|
// Start tracking when both camera and model are ready
|
|
useEffect(() => {
|
|
if (cameraReady && !isModelLoading) {
|
|
startTracking();
|
|
}
|
|
}, [cameraReady, isModelLoading, startTracking]);
|
|
|
|
// Calculate styles based on tracking data
|
|
const getAvatarStyle = () => {
|
|
// Deadzone for jitter reduction
|
|
const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val;
|
|
|
|
const rX = smooth(trackingData.rotationX); // Pitch
|
|
const rY = smooth(trackingData.rotationY); // Yaw
|
|
const rZ = smooth(trackingData.rotationZ); // Roll
|
|
const tX = smooth(trackingData.translationX);
|
|
const tY = smooth(trackingData.translationY);
|
|
|
|
// Bounce effect on mouth open (Speaking emulation)
|
|
const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0;
|
|
|
|
return {
|
|
transform: `
|
|
translate(${tX * 150}px, ${tY * 100 + bounce}px)
|
|
rotate(${rZ * 1}rad)
|
|
perspective(500px)
|
|
rotateX(${rX * 15}deg)
|
|
rotateY(${rY * -25}deg)
|
|
scale(${1 + trackingData.mouthOpen * 0.02})
|
|
`,
|
|
filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`, // Slight flash when speaking
|
|
transition: 'transform 0.1s ease-out, filter 0.1s ease'
|
|
};
|
|
};
|
|
|
|
return (
|
|
<div className="h-screen w-full flex flex-col bg-slate-900 overflow-hidden relative">
|
|
{/* Hidden Video Element for Tracking */}
|
|
<video
|
|
ref={videoRef}
|
|
autoPlay
|
|
playsInline
|
|
muted
|
|
className="absolute opacity-0 pointer-events-none w-1 h-1"
|
|
/>
|
|
|
|
{/* Top Bar */}
|
|
<div className="absolute top-0 left-0 right-0 z-20 p-4 flex justify-between items-center bg-gradient-to-b from-slate-900 to-transparent">
|
|
<button
|
|
onClick={onBack}
|
|
className="px-4 py-2 bg-slate-800/80 hover:bg-slate-700 backdrop-blur rounded-lg text-white font-medium transition-colors border border-slate-600"
|
|
>
|
|
← Exit Studio
|
|
</button>
|
|
<div className="flex gap-2">
|
|
<div className={`px-3 py-1 rounded-full text-xs font-bold flex items-center gap-2 ${isModelLoading ? 'bg-yellow-500/20 text-yellow-400' : 'bg-green-500/20 text-green-400'}`}>
|
|
<span className={`w-2 h-2 rounded-full ${isModelLoading ? 'bg-yellow-400 animate-pulse' : 'bg-green-400'}`}></span>
|
|
{isModelLoading ? 'Loading Vision Model...' : 'Tracking Active'}
|
|
</div>
|
|
<div className="px-3 py-1 rounded-full text-xs font-bold bg-purple-500/20 text-purple-400 border border-purple-500/30">
|
|
{avatar.name}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Main Stage */}
|
|
<div className="flex-1 relative flex items-center justify-center overflow-hidden">
|
|
{/* Background Grid/Effect */}
|
|
<div className="absolute inset-0 opacity-20"
|
|
style={{
|
|
backgroundImage: 'radial-gradient(#4f46e5 1px, transparent 1px)',
|
|
backgroundSize: '30px 30px'
|
|
}}>
|
|
</div>
|
|
|
|
<div className="absolute inset-0 bg-gradient-to-t from-slate-900 via-transparent to-slate-900 pointer-events-none"></div>
|
|
|
|
{/* Avatar Container */}
|
|
<div className="relative w-[600px] h-[600px] flex items-center justify-center z-10">
|
|
<div
|
|
className="relative w-full h-full flex items-center justify-center"
|
|
style={getAvatarStyle()}
|
|
>
|
|
{/* Main Character Body */}
|
|
<img
|
|
src={avatar.imageUrl}
|
|
alt="Avatar"
|
|
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
|
|
style={{
|
|
// Use clip-path to hide the right-side assets from the main view, keeping only the main character
|
|
clipPath: 'inset(0 25% 0 0)' // Hides the right 25% (where assets are)
|
|
}}
|
|
/>
|
|
|
|
{/* Dynamic Eyelids (High Fidelity Sprites) */}
|
|
{avatar.leftEye && avatar.textureClosedEye && (
|
|
<Sprite
|
|
imageSrc={avatar.imageUrl}
|
|
sourceRect={avatar.textureClosedEye}
|
|
className="absolute pointer-events-none z-20"
|
|
style={{
|
|
left: `${avatar.leftEye.x * 100}%`,
|
|
top: `${avatar.leftEye.y * 100}%`,
|
|
width: `${avatar.leftEye.w * 100}%`,
|
|
height: `${avatar.leftEye.h * 100}%`,
|
|
opacity: trackingData.isBlinkingLeft ? 1 : 0,
|
|
transition: 'opacity 0.05s linear',
|
|
}}
|
|
/>
|
|
)}
|
|
|
|
{avatar.rightEye && avatar.textureClosedEye && (
|
|
<Sprite
|
|
imageSrc={avatar.imageUrl}
|
|
sourceRect={avatar.textureClosedEye}
|
|
className="absolute pointer-events-none z-20"
|
|
style={{
|
|
left: `${avatar.rightEye.x * 100}%`,
|
|
top: `${avatar.rightEye.y * 100}%`,
|
|
width: `${avatar.rightEye.w * 100}%`,
|
|
height: `${avatar.rightEye.h * 100}%`,
|
|
opacity: trackingData.isBlinkingRight ? 1 : 0,
|
|
transition: 'opacity 0.05s linear',
|
|
}}
|
|
/>
|
|
)}
|
|
|
|
{/* Dynamic Mouth Animation */}
|
|
{avatar.mouth && avatar.textureOpenMouth && (
|
|
<div
|
|
className="absolute pointer-events-none flex items-center justify-center z-10"
|
|
style={{
|
|
left: `${avatar.mouth.x * 100}%`,
|
|
top: `${avatar.mouth.y * 100}%`,
|
|
width: `${avatar.mouth.w * 100}%`,
|
|
height: `${avatar.mouth.h * 100}%`,
|
|
}}
|
|
>
|
|
{/* Skin Patch - Hides the static closed mouth when speaking */}
|
|
<div
|
|
className="absolute w-[120%] h-[120%] transition-opacity duration-75"
|
|
style={{
|
|
backgroundColor: avatar.skinColor || '#fcd3bf',
|
|
opacity: trackingData.mouthOpen > 0.1 ? 1 : 0,
|
|
filter: 'blur(4px)', // Blends edges
|
|
borderRadius: '50%'
|
|
}}
|
|
/>
|
|
|
|
{/* Mouth Sprite - Scales based on mouth openness */}
|
|
<Sprite
|
|
imageSrc={avatar.imageUrl}
|
|
sourceRect={avatar.textureOpenMouth}
|
|
className="w-full h-full"
|
|
style={{
|
|
opacity: trackingData.mouthOpen > 0.05 ? 1 : 0,
|
|
// Scale open mouth based on volume
|
|
transform: `scaleY(${0.8 + trackingData.mouthOpen * 0.5})`,
|
|
}}
|
|
/>
|
|
</div>
|
|
)}
|
|
</div>
|
|
|
|
{/* Status Indicator overlay if tracking is lost */}
|
|
{(!cameraReady) && (
|
|
<div className="absolute inset-0 flex items-center justify-center bg-slate-900/80 z-20 rounded-xl backdrop-blur-sm">
|
|
<div className="text-cyan-400 animate-pulse font-mono">INITIALIZING CAMERA LINK...</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
{/* Control Deck */}
|
|
<div className="h-24 bg-slate-800 border-t border-slate-700 p-4 flex justify-center items-center gap-6 z-20">
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">MOUTH</span>
|
|
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden">
|
|
<div className="h-full bg-cyan-400 transition-all duration-75" style={{ width: `${Math.min(trackingData.mouthOpen * 100, 100)}%` }}></div>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">HEAD ROLL</span>
|
|
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden flex justify-center relative">
|
|
{/* Center marker */}
|
|
<div className="absolute w-[1px] h-full bg-slate-500 left-1/2"></div>
|
|
<div
|
|
className="h-full bg-purple-500 transition-all duration-75 absolute"
|
|
style={{
|
|
width: `${Math.abs(trackingData.rotationZ * 50)}%`,
|
|
left: trackingData.rotationZ < 0 ? 'auto' : '50%',
|
|
right: trackingData.rotationZ < 0 ? '50%' : 'auto'
|
|
}}
|
|
></div>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">BLINK</span>
|
|
<div className="flex gap-2">
|
|
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingLeft ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
|
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingRight ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
);
|
|
};
|
|
|
|
export default Studio;
|