working on the segmenting
This commit is contained in:
parent
917579acca
commit
bb927b0904
@ -1,5 +1,5 @@
|
||||
import React, { useState } from 'react';
|
||||
import { stitchAssets, fileToDataUrl } from '../services/imageService';
|
||||
import { stitchAssets, fileToDataUrl, sliceAvatarSheetAsync } from '../services/imageService';
|
||||
import { generateAvatarImage } from '../services/geminiService';
|
||||
import LoadingSpinner from './LoadingSpinner';
|
||||
|
||||
@ -35,7 +35,12 @@ const AvatarCreator: React.FC<AvatarCreatorProps> = ({ onAvatarGenerated }) => {
|
||||
try {
|
||||
const imageUrl = await generateAvatarImage(prompt);
|
||||
|
||||
// No automatic analysis - user will rig expressions manually
|
||||
// Automate extraction of expression parts
|
||||
const slices = await sliceAvatarSheetAsync(imageUrl);
|
||||
console.log('Automated slices generated:', slices);
|
||||
|
||||
// We still use the full sheet for the RiggingEditor,
|
||||
// but the slices are now available for future use (e.g. saving as separate files)
|
||||
onAvatarGenerated(imageUrl, name, {});
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
|
||||
@ -182,18 +182,20 @@ const Studio: React.FC<StudioProps> = ({ avatar, onBack }) => {
|
||||
const calculateFeaturePosition = (featureRect: Rect, featureType: 'eye' | 'mouth') => {
|
||||
if (!avatar.riggingReference || !featureRect) return { x: 0, y: 0 };
|
||||
|
||||
const { faceCenter, faceWidth, faceHeight } = avatar.riggingReference;
|
||||
const { faceCenter } = avatar.riggingReference;
|
||||
|
||||
// Calculate feature position relative to face center in rigging space
|
||||
// Calculate feature position relative to face center in rigging space (normalized)
|
||||
const featureCenterX = featureRect.x + featureRect.w / 2;
|
||||
const featureCenterY = featureRect.y + featureRect.h / 2;
|
||||
|
||||
const relX = featureCenterX - faceCenter.x;
|
||||
const relY = featureCenterY - faceCenter.y;
|
||||
|
||||
// Scale relative positions by face width/height to match tracking scale
|
||||
const scaledX = relX * faceWidth * avatarPosition.scale;
|
||||
const scaledY = relY * faceHeight * avatarPosition.scale;
|
||||
// Map normalized 0-1 rigging space to the visual scale of the avatar's "face size"
|
||||
// Since the container is 600px, and the face usually takes up a good chunk of it,
|
||||
// multiplying by 600 allows the relative positions to shift the features correctly.
|
||||
const scaledX = relX * 600 * avatarPosition.scale;
|
||||
const scaledY = relY * 600 * avatarPosition.scale;
|
||||
|
||||
return {
|
||||
x: scaledX,
|
||||
|
||||
@ -19,6 +19,105 @@ export const loadImage = (src: string): Promise<HTMLImageElement> => {
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Extracts a specific rectangle from an image and returns it as a data URL.
|
||||
*/
|
||||
export const extractRect = async (image: HTMLImageElement, rect: Rect): Promise<string> => {
|
||||
const canvas = document.createElement('canvas');
|
||||
const x = rect.x * image.width;
|
||||
const y = rect.y * image.height;
|
||||
const w = rect.w * image.width;
|
||||
const h = rect.h * image.height;
|
||||
|
||||
canvas.width = w;
|
||||
canvas.height = h;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) throw new Error("Could not get canvas context");
|
||||
|
||||
ctx.drawImage(image, x, y, w, h, 0, 0, w, h);
|
||||
return canvas.toDataURL('image/png');
|
||||
};
|
||||
|
||||
/**
|
||||
* Automatically slices the AI-generated sprite sheet into separate files based on the EXPRESSION_SYSTEM.md grid.
|
||||
*
|
||||
* Grid Layout:
|
||||
* Row 1: Base Character (Full Width)
|
||||
* Row 2: Eye Expressions (6 equal cols)
|
||||
* Row 3: Mouth Expressions (6 equal cols)
|
||||
*/
|
||||
export const sliceAvatarSheet = async (imageUrl: string) => {
|
||||
const image = await loadImage(imageUrl);
|
||||
const w = image.width;
|
||||
const h = image.height;
|
||||
|
||||
// Based on the 3-row grid described in EXPRESSION_SYSTEM.md
|
||||
// We assume equal height for rows, though the base character might be larger.
|
||||
// In a standard sprite sheet, let's assume rows are roughly divided.
|
||||
// Row 1 (Base), Row 2 (Eyes), Row 3 (Mouth)
|
||||
const rowH = h / 3;
|
||||
const colW = w / 6;
|
||||
|
||||
const slices: Record<string, string> = {};
|
||||
|
||||
// 1. Base Face (Full width of Row 1)
|
||||
slices['base'] = await extractRect(image, { x: 0, y: 0, w: 1, h: 1/3 });
|
||||
|
||||
// 2. Eye Expressions (Row 2)
|
||||
const eyeTypes = ['NEUTRAL', 'HAPPY', 'SURPRISED', 'ANGRY', 'SAD', 'BLINK'];
|
||||
eyeTypes.forEach((type, i) => {
|
||||
slices[`eye_${type}`] = extractRect(image, {
|
||||
x: i * (1/6),
|
||||
y: 1/3,
|
||||
w: 1/6,
|
||||
h: 1/3
|
||||
}) as any; // In a real loop we should await or Promise.all
|
||||
});
|
||||
|
||||
// 3. Mouth Expressions (Row 3)
|
||||
const mouthTypes = ['NEUTRAL', 'HAPPY', 'OPEN_TALK', 'WIDE_OPEN', 'FROWN', 'O_SHAPE'];
|
||||
mouthTypes.forEach((type, i) => {
|
||||
slices[`mouth_${type}`] = extractRect(image, {
|
||||
x: i * (1/6),
|
||||
y: 2/3,
|
||||
w: 1/6,
|
||||
h: 1/3
|
||||
}) as any;
|
||||
});
|
||||
|
||||
// Since the above are promises, we need to wrap this properly.
|
||||
// See implementation below.
|
||||
return slices;
|
||||
};
|
||||
|
||||
// Refined slice function to handle promises properly
|
||||
export const sliceAvatarSheetAsync = async (imageUrl: string) => {
|
||||
const image = await loadImage(imageUrl);
|
||||
const rowH = 1/3;
|
||||
const colW = 1/6;
|
||||
|
||||
const eyeTypes = ['NEUTRAL', 'HAPPY', 'SURPRISED', 'ANGRY', 'SAD', 'BLINK'];
|
||||
const mouthTypes = ['NEUTRAL', 'HAPPY', 'OPEN_TALK', 'WIDE_OPEN', 'FROWN', 'O_SHAPE'];
|
||||
|
||||
const results: Record<string, string> = {};
|
||||
|
||||
results['base'] = await extractRect(image, { x: 0, y: 0, w: 1, h: rowH });
|
||||
|
||||
for (let i = 0; i < eyeTypes.length; i++) {
|
||||
results[`eye_${eyeTypes[i]}`] = await extractRect(image, {
|
||||
x: i * colW, y: rowH, w: colW, h: rowH
|
||||
});
|
||||
}
|
||||
|
||||
for (let i = 0; i < mouthTypes.length; i++) {
|
||||
results[`mouth_${mouthTypes[i]}`] = await extractRect(image, {
|
||||
x: i * colW, y: rowH * 2, w: colW, h: rowH
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
};
|
||||
|
||||
export const stitchAssets = async (
|
||||
baseSrc: string,
|
||||
blinkSrc?: string,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user