import React, { useEffect, useRef, useState } from 'react'; import { useFaceTracking } from '../hooks/useFaceTracking'; import { removeBackground } from '../services/visionService'; import { AvatarConfig, Rect } from '../types'; import LoadingSpinner from './LoadingSpinner'; interface StudioProps { avatar: AvatarConfig; onBack: () => void; } /** * Sprite Component * Renders a specific crop of the source image into a target container. */ const Sprite: React.FC<{ imageSrc: string; sourceRect: Rect; style?: React.CSSProperties; className?: string; }> = ({ imageSrc, sourceRect, style, className }) => { // To display a cropped region (sourceRect) of the image, we use an inner // positioned negatively and scaled up. // Example: If sourceRect.w is 0.1 (10%), the image must be scaled to 10x (1000%) size. const widthScale = 100 / (sourceRect.w * 100); const heightScale = 100 / (sourceRect.h * 100); return (
); }; const Studio: React.FC = ({ avatar, onBack }) => { const videoRef = useRef(null); const [cameraReady, setCameraReady] = useState(false); const [processedImageUrl, setProcessedImageUrl] = useState(null); // We use the custom hook to get tracking data const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current); // Initialize Camera useEffect(() => { const startCamera = async () => { try { const stream = await navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 480 }, // Lower res is fine for tracking audio: false }); if (videoRef.current) { videoRef.current.srcObject = stream; videoRef.current.onloadeddata = () => { setCameraReady(true); }; } } catch (err) { console.error("Error accessing camera:", err); alert("Could not access camera. Please ensure permissions are granted."); } }; startCamera(); return () => { // Cleanup stream if (videoRef.current && videoRef.current.srcObject) { const stream = videoRef.current.srcObject as MediaStream; stream.getTracks().forEach(track => track.stop()); } }; }, []); // Process Image for Background Removal (AI Segmentation) useEffect(() => { if (!avatar.chromaKeyColor) { setProcessedImageUrl(avatar.imageUrl); return; } const process = async () => { // If chromaKeyColor is set (to anything, now treated as a flag), we run AI removal const result = await removeBackground(avatar.imageUrl); setProcessedImageUrl(result); }; process(); }, [avatar.imageUrl, avatar.chromaKeyColor]); // Start tracking when both camera and model are ready useEffect(() => { if (cameraReady && !isModelLoading) { startTracking(); } }, [cameraReady, isModelLoading, startTracking]); // Calculate styles based on tracking data const getAvatarStyle = () => { // Deadzone for jitter reduction const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val; const rX = smooth(trackingData.rotationX); // Pitch const rY = smooth(trackingData.rotationY); // Yaw const rZ = smooth(trackingData.rotationZ); // Roll const tX = smooth(trackingData.translationX); const tY = smooth(trackingData.translationY); // Bounce effect on mouth open (Speaking emulation) const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0; return { transform: ` translate(${tX * 150}px, ${tY * 100 + bounce}px) rotate(${rZ * 1}rad) perspective(500px) rotateX(${rX * 15}deg) rotateY(${rY * -25}deg) scale(${1 + trackingData.mouthOpen * 0.02}) `, filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`, // Slight flash when speaking transition: 'transform 0.1s ease-out, filter 0.1s ease' }; }; return (
{/* Hidden Video Element for Tracking */}