ddb2455416
Enables users to upload custom avatar assets and automatically remove the background from the generated image. New features: - Avatar creation now supports uploading base, blink, and talk textures. - Added ability to define the main body bounding box during rigging. - Vision service now includes image segmentation for background removal. - Studio component dynamically processes the avatar image for background removal if chroma key is enabled.
330 lines
13 KiB
TypeScript
330 lines
13 KiB
TypeScript
|
|
import React, { useEffect, useRef, useState } from 'react';
|
|
import { useFaceTracking } from '../hooks/useFaceTracking';
|
|
import { removeBackground } from '../services/visionService';
|
|
import { AvatarConfig, Rect } from '../types';
|
|
import LoadingSpinner from './LoadingSpinner';
|
|
|
|
interface StudioProps {
|
|
avatar: AvatarConfig;
|
|
onBack: () => void;
|
|
}
|
|
|
|
/**
|
|
* Sprite Component
|
|
* Renders a specific crop of the source image into a target container.
|
|
*/
|
|
const Sprite: React.FC<{
|
|
imageSrc: string;
|
|
sourceRect: Rect;
|
|
style?: React.CSSProperties;
|
|
className?: string;
|
|
}> = ({ imageSrc, sourceRect, style, className }) => {
|
|
// To display a cropped region (sourceRect) of the image, we use an inner <img>
|
|
// positioned negatively and scaled up.
|
|
// Example: If sourceRect.w is 0.1 (10%), the image must be scaled to 10x (1000%) size.
|
|
const widthScale = 100 / (sourceRect.w * 100);
|
|
const heightScale = 100 / (sourceRect.h * 100);
|
|
|
|
return (
|
|
<div
|
|
className={`overflow-hidden relative ${className}`}
|
|
style={style}
|
|
>
|
|
<img
|
|
src={imageSrc}
|
|
alt=""
|
|
style={{
|
|
position: 'absolute',
|
|
top: `-${sourceRect.y * 100 * heightScale}%`,
|
|
left: `-${sourceRect.x * 100 * widthScale}%`,
|
|
width: `${widthScale * 100}%`,
|
|
height: `${heightScale * 100}%`,
|
|
maxWidth: 'none',
|
|
maxHeight: 'none',
|
|
pointerEvents: 'none'
|
|
}}
|
|
/>
|
|
</div>
|
|
);
|
|
};
|
|
|
|
const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
|
|
const videoRef = useRef<HTMLVideoElement>(null);
|
|
const [cameraReady, setCameraReady] = useState(false);
|
|
const [processedImageUrl, setProcessedImageUrl] = useState<string | null>(null);
|
|
|
|
// We use the custom hook to get tracking data
|
|
const { trackingData, isLoading: isModelLoading, startTracking } = useFaceTracking(videoRef.current);
|
|
|
|
// Initialize Camera
|
|
useEffect(() => {
|
|
const startCamera = async () => {
|
|
try {
|
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
video: { width: 640, height: 480 }, // Lower res is fine for tracking
|
|
audio: false
|
|
});
|
|
if (videoRef.current) {
|
|
videoRef.current.srcObject = stream;
|
|
videoRef.current.onloadeddata = () => {
|
|
setCameraReady(true);
|
|
};
|
|
}
|
|
} catch (err) {
|
|
console.error("Error accessing camera:", err);
|
|
alert("Could not access camera. Please ensure permissions are granted.");
|
|
}
|
|
};
|
|
|
|
startCamera();
|
|
|
|
return () => {
|
|
// Cleanup stream
|
|
if (videoRef.current && videoRef.current.srcObject) {
|
|
const stream = videoRef.current.srcObject as MediaStream;
|
|
stream.getTracks().forEach(track => track.stop());
|
|
}
|
|
};
|
|
}, []);
|
|
|
|
// Process Image for Background Removal (AI Segmentation)
|
|
useEffect(() => {
|
|
if (!avatar.chromaKeyColor) {
|
|
setProcessedImageUrl(avatar.imageUrl);
|
|
return;
|
|
}
|
|
|
|
const process = async () => {
|
|
// If chromaKeyColor is set (to anything, now treated as a flag), we run AI removal
|
|
const result = await removeBackground(avatar.imageUrl);
|
|
setProcessedImageUrl(result);
|
|
};
|
|
|
|
process();
|
|
}, [avatar.imageUrl, avatar.chromaKeyColor]);
|
|
|
|
// Start tracking when both camera and model are ready
|
|
useEffect(() => {
|
|
if (cameraReady && !isModelLoading) {
|
|
startTracking();
|
|
}
|
|
}, [cameraReady, isModelLoading, startTracking]);
|
|
|
|
// Calculate styles based on tracking data
|
|
const getAvatarStyle = () => {
|
|
// Deadzone for jitter reduction
|
|
const smooth = (val: number) => Math.abs(val) < 0.02 ? 0 : val;
|
|
|
|
const rX = smooth(trackingData.rotationX); // Pitch
|
|
const rY = smooth(trackingData.rotationY); // Yaw
|
|
const rZ = smooth(trackingData.rotationZ); // Roll
|
|
const tX = smooth(trackingData.translationX);
|
|
const tY = smooth(trackingData.translationY);
|
|
|
|
// Bounce effect on mouth open (Speaking emulation)
|
|
const bounce = trackingData.mouthOpen > 0.1 ? -5 * trackingData.mouthOpen : 0;
|
|
|
|
return {
|
|
transform: `
|
|
translate(${tX * 150}px, ${tY * 100 + bounce}px)
|
|
rotate(${rZ * 1}rad)
|
|
perspective(500px)
|
|
rotateX(${rX * 15}deg)
|
|
rotateY(${rY * -25}deg)
|
|
scale(${1 + trackingData.mouthOpen * 0.02})
|
|
`,
|
|
filter: `brightness(${1 + trackingData.mouthOpen * 0.05})`, // Slight flash when speaking
|
|
transition: 'transform 0.1s ease-out, filter 0.1s ease'
|
|
};
|
|
};
|
|
|
|
return (
|
|
<div className="h-screen w-full flex flex-col bg-slate-900 overflow-hidden relative">
|
|
{/* Hidden Video Element for Tracking */}
|
|
<video
|
|
ref={videoRef}
|
|
autoPlay
|
|
playsInline
|
|
muted
|
|
className="absolute opacity-0 pointer-events-none w-1 h-1"
|
|
/>
|
|
|
|
{/* Top Bar */}
|
|
<div className="absolute top-0 left-0 right-0 z-20 p-4 flex justify-between items-center bg-gradient-to-b from-slate-900 to-transparent">
|
|
<button
|
|
onClick={onBack}
|
|
className="px-4 py-2 bg-slate-800/80 hover:bg-slate-700 backdrop-blur rounded-lg text-white font-medium transition-colors border border-slate-600"
|
|
>
|
|
← Exit Studio
|
|
</button>
|
|
<div className="flex gap-2">
|
|
<div className={`px-3 py-1 rounded-full text-xs font-bold flex items-center gap-2 ${isModelLoading ? 'bg-yellow-500/20 text-yellow-400' : 'bg-green-500/20 text-green-400'}`}>
|
|
<span className={`w-2 h-2 rounded-full ${isModelLoading ? 'bg-yellow-400 animate-pulse' : 'bg-green-400'}`}></span>
|
|
{isModelLoading ? 'Loading Vision Model...' : 'Tracking Active'}
|
|
</div>
|
|
<div className="px-3 py-1 rounded-full text-xs font-bold bg-purple-500/20 text-purple-400 border border-purple-500/30">
|
|
{avatar.name}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Main Stage */}
|
|
<div className="flex-1 relative flex items-center justify-center overflow-hidden">
|
|
{/* Background Grid/Effect */}
|
|
<div className="absolute inset-0 opacity-20"
|
|
style={{
|
|
backgroundImage: 'radial-gradient(#4f46e5 1px, transparent 1px)',
|
|
backgroundSize: '30px 30px'
|
|
}}>
|
|
</div>
|
|
|
|
<div className="absolute inset-0 bg-gradient-to-t from-slate-900 via-transparent to-slate-900 pointer-events-none"></div>
|
|
|
|
{/* Avatar Container */}
|
|
<div className="relative w-[600px] h-[600px] flex items-center justify-center z-10">
|
|
{!processedImageUrl ? (
|
|
<div className="flex flex-col items-center justify-center gap-4">
|
|
<LoadingSpinner />
|
|
<span className="text-cyan-400 font-mono text-sm">REMOVING BACKGROUND...</span>
|
|
</div>
|
|
) : (
|
|
<div
|
|
className="relative w-full h-full flex items-center justify-center"
|
|
style={getAvatarStyle()}
|
|
>
|
|
{/* Main Character Body (Cropped using Sprite) */}
|
|
{avatar.mainBody ? (
|
|
<Sprite
|
|
imageSrc={processedImageUrl}
|
|
sourceRect={avatar.mainBody}
|
|
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
|
|
/>
|
|
) : (
|
|
/* Fallback to full image if mainBody is missing */
|
|
<img
|
|
src={processedImageUrl}
|
|
alt="Avatar"
|
|
className="w-full h-full object-contain drop-shadow-[0_0_15px_rgba(168,85,247,0.5)]"
|
|
/>
|
|
)}
|
|
|
|
{/* Dynamic Eyelids (High Fidelity Sprites) */}
|
|
{avatar.leftEye && avatar.textureClosedEye && (
|
|
<Sprite
|
|
imageSrc={processedImageUrl}
|
|
sourceRect={avatar.textureClosedEye}
|
|
className="absolute pointer-events-none z-20"
|
|
style={{
|
|
left: `${avatar.leftEye.x * 100}%`,
|
|
top: `${avatar.leftEye.y * 100}%`,
|
|
width: `${avatar.leftEye.w * 100}%`,
|
|
height: `${avatar.leftEye.h * 100}%`,
|
|
opacity: trackingData.isBlinkingLeft ? 1 : 0,
|
|
transition: 'opacity 0.05s linear',
|
|
}}
|
|
/>
|
|
)}
|
|
|
|
{avatar.rightEye && avatar.textureClosedEye && (
|
|
<Sprite
|
|
imageSrc={processedImageUrl}
|
|
sourceRect={avatar.textureClosedEye}
|
|
className="absolute pointer-events-none z-20"
|
|
style={{
|
|
left: `${avatar.rightEye.x * 100}%`,
|
|
top: `${avatar.rightEye.y * 100}%`,
|
|
width: `${avatar.rightEye.w * 100}%`,
|
|
height: `${avatar.rightEye.h * 100}%`,
|
|
opacity: trackingData.isBlinkingRight ? 1 : 0,
|
|
transition: 'opacity 0.05s linear',
|
|
}}
|
|
/>
|
|
)}
|
|
|
|
{/* Dynamic Mouth Animation */}
|
|
{avatar.mouth && avatar.textureOpenMouth && (
|
|
<div
|
|
className="absolute pointer-events-none flex items-center justify-center z-10"
|
|
style={{
|
|
left: `${avatar.mouth.x * 100}%`,
|
|
top: `${avatar.mouth.y * 100}%`,
|
|
width: `${avatar.mouth.w * 100}%`,
|
|
height: `${avatar.mouth.h * 100}%`,
|
|
}}
|
|
>
|
|
{/* Skin Patch - Hides the static closed mouth when speaking */}
|
|
<div
|
|
className="absolute w-[120%] h-[120%] transition-opacity duration-75"
|
|
style={{
|
|
backgroundColor: avatar.skinColor || '#fcd3bf',
|
|
opacity: trackingData.mouthOpen > 0.1 ? 1 : 0,
|
|
filter: 'blur(4px)', // Blends edges
|
|
borderRadius: '50%'
|
|
}}
|
|
/>
|
|
|
|
{/* Mouth Sprite - Scales based on mouth openness */}
|
|
<Sprite
|
|
imageSrc={processedImageUrl}
|
|
sourceRect={avatar.textureOpenMouth}
|
|
className="w-full h-full"
|
|
style={{
|
|
opacity: trackingData.mouthOpen > 0.05 ? 1 : 0,
|
|
// Scale open mouth based on volume
|
|
transform: `scaleY(${0.8 + trackingData.mouthOpen * 0.5})`,
|
|
}}
|
|
/>
|
|
</div>
|
|
)}
|
|
</div>
|
|
)}
|
|
|
|
{/* Status Indicator overlay if tracking is lost */}
|
|
{(!cameraReady) && (
|
|
<div className="absolute inset-0 flex items-center justify-center bg-slate-900/80 z-20 rounded-xl backdrop-blur-sm">
|
|
<div className="text-cyan-400 animate-pulse font-mono">INITIALIZING CAMERA LINK...</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
{/* Control Deck */}
|
|
<div className="h-24 bg-slate-800 border-t border-slate-700 p-4 flex justify-center items-center gap-6 z-20">
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">MOUTH</span>
|
|
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden">
|
|
<div className="h-full bg-cyan-400 transition-all duration-75" style={{ width: `${Math.min(trackingData.mouthOpen * 100, 100)}%` }}></div>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">HEAD ROLL</span>
|
|
<div className="w-24 h-2 bg-slate-700 rounded-full overflow-hidden flex justify-center relative">
|
|
{/* Center marker */}
|
|
<div className="absolute w-[1px] h-full bg-slate-500 left-1/2"></div>
|
|
<div
|
|
className="h-full bg-purple-500 transition-all duration-75 absolute"
|
|
style={{
|
|
width: `${Math.abs(trackingData.rotationZ * 50)}%`,
|
|
left: trackingData.rotationZ < 0 ? 'auto' : '50%',
|
|
right: trackingData.rotationZ < 0 ? '50%' : 'auto'
|
|
}}
|
|
></div>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex flex-col items-center">
|
|
<span className="text-xs text-slate-400 mb-1 font-mono">BLINK</span>
|
|
<div className="flex gap-2">
|
|
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingLeft ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
|
<div className={`w-8 h-2 rounded-full ${trackingData.isBlinkingRight ? 'bg-pink-500' : 'bg-slate-700'}`}></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
);
|
|
};
|
|
|
|
export default Studio;
|