diff --git a/App.tsx b/App.tsx index f10f169..f320ef6 100644 --- a/App.tsx +++ b/App.tsx @@ -31,7 +31,10 @@ const App: React.FC = () => { setAppState(AppState.RIGGING); }; - const handleRiggingComplete = (data: { leftEye: Rect, rightEye: Rect, mouth: Rect, skinColor: string }) => { + const handleRiggingComplete = (data: { + leftEye: Rect, rightEye: Rect, mouth: Rect, skinColor: string, + textureClosedEye: Rect, textureOpenMouth: Rect + }) => { if (generatedData) { setAvatar({ imageUrl: generatedData.url, @@ -40,7 +43,9 @@ const App: React.FC = () => { leftEye: data.leftEye, rightEye: data.rightEye, mouth: data.mouth, - skinColor: data.skinColor + skinColor: data.skinColor, + textureClosedEye: data.textureClosedEye, + textureOpenMouth: data.textureOpenMouth, }); setAppState(AppState.STUDIO); } diff --git a/components/AvatarCreator.tsx b/components/AvatarCreator.tsx index 92403e1..1852bd1 100644 --- a/components/AvatarCreator.tsx +++ b/components/AvatarCreator.tsx @@ -22,10 +22,11 @@ const AvatarCreator: React.FC = ({ onAvatarGenerated }) => { setError(null); try { - // 1. Generate Image + // 1. Generate Image (Now creates a character sheet) const imageUrl = await generateAvatarImage(prompt); // 2. Analyze Image for Landmarks (Initial guess) + // Note: Vision service will likely find the main face on the left, which is what we want for targets. setStatus('analyzing'); const analysisData = await analyzeAvatarImage(imageUrl); @@ -50,7 +51,7 @@ const AvatarCreator: React.FC = ({ onAvatarGenerated }) => { Design Your Avatar

- Describe your dream VTuber model and let Gemini bring it to life. + Describe your dream VTuber model. Gemini will generate a character sheet with expression assets.

@@ -94,7 +95,7 @@ const AvatarCreator: React.FC = ({ onAvatarGenerated }) => { {status !== 'idle' ? (
- {status === 'generating' ? 'Dreaming up Avatar...' : 'Analyzing Features...'} + {status === 'generating' ? 'Dreaming up Sheet...' : 'Analyzing Features...'}
) : (
diff --git a/components/RiggingEditor.tsx b/components/RiggingEditor.tsx index 4e23c3f..af4102b 100644 --- a/components/RiggingEditor.tsx +++ b/components/RiggingEditor.tsx @@ -5,10 +5,13 @@ import { Rect } from '../types'; interface RiggingEditorProps { imageUrl: string; initialData?: { leftEye: Rect; rightEye: Rect; mouth: Rect; skinColor: string }; - onComplete: (data: { leftEye: Rect; rightEye: Rect; mouth: Rect; skinColor: string }) => void; + onComplete: (data: { + leftEye: Rect; rightEye: Rect; mouth: Rect; skinColor: string; + textureClosedEye: Rect; textureOpenMouth: Rect; + }) => void; } -type ActiveFeature = 'leftEye' | 'rightEye' | 'mouth' | null; +type ActiveFeature = 'leftEye' | 'rightEye' | 'mouth' | 'textureClosedEye' | 'textureOpenMouth' | null; const ResizableBox: React.FC<{ rect: Rect; @@ -88,7 +91,7 @@ const ResizableBox: React.FC<{
{/* Label */}
{label} @@ -117,100 +120,114 @@ const ResizableBox: React.FC<{ }; const RiggingEditor: React.FC = ({ imageUrl, initialData, onComplete }) => { - const [leftEye, setLeftEye] = useState(initialData?.leftEye || { x: 0.35, y: 0.4, w: 0.12, h: 0.08 }); - const [rightEye, setRightEye] = useState(initialData?.rightEye || { x: 0.53, y: 0.4, w: 0.12, h: 0.08 }); - const [mouth, setMouth] = useState(initialData?.mouth || { x: 0.45, y: 0.6, w: 0.1, h: 0.05 }); + // Targets (Left side of image usually) + const [leftEye, setLeftEye] = useState(initialData?.leftEye || { x: 0.25, y: 0.4, w: 0.1, h: 0.1 }); + const [rightEye, setRightEye] = useState(initialData?.rightEye || { x: 0.45, y: 0.4, w: 0.1, h: 0.1 }); + const [mouth, setMouth] = useState(initialData?.mouth || { x: 0.35, y: 0.55, w: 0.1, h: 0.05 }); + + // Sources (Right side of image usually) + const [textureClosedEye, setTextureClosedEye] = useState({ x: 0.7, y: 0.1, w: 0.2, h: 0.2 }); + const [textureOpenMouth, setTextureOpenMouth] = useState({ x: 0.7, y: 0.5, w: 0.2, h: 0.2 }); + const [skinColor, setSkinColor] = useState(initialData?.skinColor || '#fcd3bf'); const [activeFeature, setActiveFeature] = useState(null); return ( -
+
-

Rig Your Avatar

-

- Drag and resize the boxes to match your avatar's features. - This ensures the eyes blink correctly. +

Rig Your Character

+

+ 1. Match the Target boxes (Red/Blue/Green) to the main character.
+ 2. Match the Source boxes (Purple/Orange) to the extra assets on the right.

-
+
{/* Editor Area */} -
-
+
+
Rigging Target - setActiveFeature('leftEye')} - /> - - setActiveFeature('rightEye')} - /> - - setActiveFeature('mouth')} - /> + {/* Aspect ratio container to map percentage boxes correctly */} +
+ {/* Targets */} + setActiveFeature('leftEye')} + /> + setActiveFeature('rightEye')} + /> + setActiveFeature('mouth')} + /> + + {/* Sources */} + setActiveFeature('textureClosedEye')} + /> + setActiveFeature('textureOpenMouth')} + /> +
{/* Sidebar Controls */} -
+
-
- +
+
setSkinColor(e.target.value)} - className="w-10 h-10 rounded cursor-pointer border-0 p-0" + className="w-8 h-8 rounded cursor-pointer border-0 p-0" /> {skinColor}
-

- Pick the color of the skin above the eyes for realistic blinking. -

-
-
-
- Left Eye Box +
+
Targets (Main Face)
+
setActiveFeature('leftEye')}> +
Left Eye
-
-
- Right Eye Box +
setActiveFeature('rightEye')}> +
Right Eye
-
-
- Mouth Box +
setActiveFeature('mouth')}> +
Mouth +
+ +
Sources (Right Side)
+
setActiveFeature('textureClosedEye')}> +
Closed Eye Texture +
+
setActiveFeature('textureOpenMouth')}> +
Open Mouth Texture
-
+
diff --git a/components/Studio.tsx b/components/Studio.tsx index e20739a..a857c9c 100644 --- a/components/Studio.tsx +++ b/components/Studio.tsx @@ -1,12 +1,52 @@ + import React, { useEffect, useRef, useState } from 'react'; import { useFaceTracking } from '../hooks/useFaceTracking'; -import { AvatarConfig } from '../types'; +import { AvatarConfig, Rect } from '../types'; interface StudioProps { avatar: AvatarConfig; onBack: () => void; } +/** + * Sprite Component + * Renders a specific crop of the source image into a target container. + */ +const Sprite: React.FC<{ + imageSrc: string; + sourceRect: Rect; + style?: React.CSSProperties; + className?: string; +}> = ({ imageSrc, sourceRect, style, className }) => { + // To display a cropped region (sourceRect) of the image, we use an inner + // positioned negatively and scaled up. + // Example: If sourceRect.w is 0.1 (10%), the image must be scaled to 10x (1000%) size. + const widthScale = 100 / (sourceRect.w * 100); + const heightScale = 100 / (sourceRect.h * 100); + + return ( +
+ +
+ ); +}; + const Studio: React.FC = ({ avatar, onBack }) => { const videoRef = useRef(null); const [cameraReady, setCameraReady] = useState(false); @@ -128,49 +168,52 @@ const Studio: React.FC = ({ avatar, onBack }) => { className="relative w-full h-full flex items-center justify-center" style={getAvatarStyle()} > + {/* Main Character Body */} Avatar - {/* Dynamic Eyelids */} - {avatar.leftEye && avatar.skinColor && ( -
)} - {avatar.rightEye && avatar.skinColor && ( -
)} {/* Dynamic Mouth Animation */} - {avatar.mouth && ( + {avatar.mouth && avatar.textureOpenMouth && (
= ({ avatar, onBack }) => { style={{ backgroundColor: avatar.skinColor || '#fcd3bf', opacity: trackingData.mouthOpen > 0.1 ? 1 : 0, - filter: 'blur(3px)', // Blends edges - borderRadius: '40%' + filter: 'blur(4px)', // Blends edges + borderRadius: '50%' }} /> - {/* Mouth Interior - Scales based on mouth openness */} -
0.05 ? 1 : 0, - }} - > - {/* Tongue */} -
-
+ {/* Mouth Sprite - Scales based on mouth openness */} + 0.05 ? 1 : 0, + // Scale open mouth based on volume + transform: `scaleY(${0.8 + trackingData.mouthOpen * 0.5})`, + }} + />
)}
- {/* Optional: Status Indicator overlay if tracking is lost (all 0s usually) or just visual flair */} + {/* Status Indicator overlay if tracking is lost */} {(!cameraReady) && (
INITIALIZING CAMERA LINK...
@@ -256,4 +295,4 @@ const Studio: React.FC = ({ avatar, onBack }) => { ); }; -export default Studio; \ No newline at end of file +export default Studio; diff --git a/services/geminiService.ts b/services/geminiService.ts index 54ba093..7a187b5 100644 --- a/services/geminiService.ts +++ b/services/geminiService.ts @@ -1,25 +1,30 @@ + import { GoogleGenAI } from "@google/genai"; /** - * Generates a VTuber avatar image based on user description. + * Generates a VTuber avatar character sheet. * Uses gemini-3-pro-image-preview for high quality. */ export const generateAvatarImage = async (description: string): Promise => { try { // Initialize client inside the function to ensure we use the most up-to-date API key - // after the user has completed the selection flow. const ai = new GoogleGenAI({ apiKey: process.env.API_KEY }); - // We construct a prompt that encourages a good format for a 2D avatar (front facing, clean background) const prompt = ` - Create a high-quality, flat 2D anime or stylized character illustration suitable for a VTuber avatar. - The character should be facing forward (front view). - The background should be a solid, single color (white or bright green) to allow for easy removal or masking. + Create a VTuber character sheet with a flat 2D anime style. + + LAYOUT: + 1. MAIN CHARACTER (Left side, takes up 70% of width): + - Front-facing view, head and shoulders. + - Neutral expression, eyes open, mouth closed. + + 2. EXPRESSION ASSETS (Right side, vertical column): + - Top: The same character's face with EYES CLOSED (for blinking). + - Bottom: The same character's face with MOUTH OPEN (for talking). Character Description: ${description} - Style: Vibrant, clean lines, detailed eyes. - Focus: Head and shoulders only. + Style: Vibrant, clean lines, solid white or green background for easy keying. `; const response = await ai.models.generateContent({ @@ -31,7 +36,7 @@ export const generateAvatarImage = async (description: string): Promise }, config: { imageConfig: { - aspectRatio: "1:1", + aspectRatio: "16:9", // Wide to fit character sheet imageSize: "1K" } } diff --git a/types.ts b/types.ts index 20c70a4..fee9fdd 100644 --- a/types.ts +++ b/types.ts @@ -21,6 +21,8 @@ export interface AvatarConfig { rightEye?: Rect; mouth?: Rect; skinColor?: string; + textureClosedEye?: Rect; + textureOpenMouth?: Rect; } export interface TrackingData {