refactor: Improve avatar asset processing

Separates asset analysis into distinct steps to accurately capture face landmarks.
Introduces `fileToDataUrl` utility and modifies `stitchAssets` to accept image source strings, reducing redundant file processing and improving clarity.
This commit is contained in:
James Twose 2025-11-20 22:03:53 +01:00
parent ddb2455416
commit 5078d67d4f
2 changed files with 69 additions and 22 deletions

View File

@ -1,7 +1,7 @@
import React, { useState } from 'react'; import React, { useState } from 'react';
import { generateAvatarImage } from '../services/geminiService'; import { generateAvatarImage } from '../services/geminiService';
import { analyzeAvatarImage } from '../services/visionService'; import { analyzeAvatarImage } from '../services/visionService';
import { stitchAssets } from '../services/imageService'; import { stitchAssets, fileToDataUrl } from '../services/imageService';
import LoadingSpinner from './LoadingSpinner'; import LoadingSpinner from './LoadingSpinner';
import { Rect } from '../types'; import { Rect } from '../types';
@ -61,23 +61,72 @@ const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
setError(null); setError(null);
try { try {
// 1. Stitch Assets into Sheet // 1. Prepare Base Image and Analyze it separately
const { imageUrl, mainBody, textureClosedEye, textureOpenMouth } = await stitchAssets(baseFile, blinkFile || undefined, talkFile || undefined); // Analyzing separately ensures we get landmarks for the main face correctly
// without interference from other faces in a stitched sheet.
const baseDataUrl = await fileToDataUrl(baseFile);
const baseAnalysis = await analyzeAvatarImage(baseDataUrl);
// 2. Analyze the Main Body part of the image // 2. Prepare and Analyze Variant Images
// Note: analyzeAvatarImage analyzes the whole image, but since we put the face on the left (or full image), let blinkDataUrl, blinkAnalysis;
// it should find the face correctly. if (blinkFile) {
setStatus('analyzing'); blinkDataUrl = await fileToDataUrl(blinkFile);
const analysisData = await analyzeAvatarImage(imageUrl); // Try to find eyes in the blink image to use as tight texture crop
blinkAnalysis = await analyzeAvatarImage(blinkDataUrl);
}
// 3. Combine manual stitch data with automatic vision data let talkDataUrl, talkAnalysis;
const initialData = { if (talkFile) {
...(analysisData || {}), talkDataUrl = await fileToDataUrl(talkFile);
// Try to find mouth in the talk image
talkAnalysis = await analyzeAvatarImage(talkDataUrl);
}
// 3. Stitch Assets into Sheet
const { imageUrl, mainBody, textureClosedEye: stitchBlinkRect, textureOpenMouth: stitchTalkRect } = await stitchAssets(baseDataUrl, blinkDataUrl, talkDataUrl);
// 4. Map Analysis Data to Stitched Coordinate Space
// Helper to map a rect from (0-1 in sub-image) to (0-1 in stitched-image)
const mapRect = (r: Rect, container: Rect) => ({
x: container.x + r.x * container.w,
y: container.y + r.y * container.h,
w: r.w * container.w,
h: r.h * container.h
});
let initialData: any = {
mainBody, mainBody,
textureClosedEye, textureClosedEye: stitchBlinkRect,
textureOpenMouth textureOpenMouth: stitchTalkRect
}; };
// Map Base Targets (Eyes, Mouth on main body)
if (baseAnalysis) {
initialData.leftEye = mapRect(baseAnalysis.leftEye, mainBody);
initialData.rightEye = mapRect(baseAnalysis.rightEye, mainBody);
initialData.mouth = mapRect(baseAnalysis.mouth, mainBody);
initialData.skinColor = baseAnalysis.skinColor;
}
// Map Source Textures (Tight crop around features if detected)
// If detections fail (e.g. eyes closed might not be detected), we fall back to the whole image (stitchBlinkRect)
if (blinkAnalysis && stitchBlinkRect) {
// Calculate a bounding box around both eyes in the blink image
const be = blinkAnalysis;
const minX = Math.min(be.leftEye.x, be.rightEye.x);
const minY = Math.min(be.leftEye.y, be.rightEye.y);
const maxX = Math.max(be.leftEye.x + be.leftEye.w, be.rightEye.x + be.rightEye.w);
const maxY = Math.max(be.leftEye.y + be.leftEye.h, be.rightEye.y + be.rightEye.h);
const eyesRect = { x: minX, y: minY, w: maxX - minX, h: maxY - minY };
initialData.textureClosedEye = mapRect(eyesRect, stitchBlinkRect);
}
if (talkAnalysis && stitchTalkRect) {
initialData.textureOpenMouth = mapRect(talkAnalysis.mouth, stitchTalkRect);
}
onAvatarGenerated(imageUrl, name, initialData); onAvatarGenerated(imageUrl, name, initialData);
} catch (err) { } catch (err) {
console.error(err); console.error(err);

View File

@ -20,16 +20,14 @@ export const loadImage = (src: string): Promise<HTMLImageElement> => {
}; };
export const stitchAssets = async ( export const stitchAssets = async (
base: File, baseSrc: string,
blink?: File, blinkSrc?: string,
talk?: File talkSrc?: string
): Promise<{ imageUrl: string; mainBody: Rect; textureClosedEye?: Rect; textureOpenMouth?: Rect }> => { ): Promise<{ imageUrl: string; mainBody: Rect; textureClosedEye?: Rect; textureOpenMouth?: Rect }> => {
// Load images // Load images
const baseData = await fileToDataUrl(base); const baseImg = await loadImage(baseSrc);
const baseImg = await loadImage(baseData); const blinkImg = blinkSrc ? await loadImage(blinkSrc) : null;
const talkImg = talkSrc ? await loadImage(talkSrc) : null;
const blinkImg = blink ? await loadImage(await fileToDataUrl(blink)) : null;
const talkImg = talk ? await loadImage(await fileToDataUrl(talk)) : null;
// Layout: Base on Left. Sidebar on Right containing Blink (top) and Talk (bottom). // Layout: Base on Left. Sidebar on Right containing Blink (top) and Talk (bottom).
// Sidebar width = max(blink.width, talk.width) // Sidebar width = max(blink.width, talk.width)
@ -38,7 +36,7 @@ export const stitchAssets = async (
// If there are no variants, just return the base image as is // If there are no variants, just return the base image as is
if (sidebarWidth === 0) { if (sidebarWidth === 0) {
return { return {
imageUrl: baseData, imageUrl: baseSrc,
mainBody: { x: 0, y: 0, w: 1, h: 1 } mainBody: { x: 0, y: 0, w: 1, h: 1 }
}; };
} }