From bb927b09042d8e74d08ff52dde1ca61a5680be4d Mon Sep 17 00:00:00 2001 From: itsamejms Date: Sun, 7 Jun 2026 17:47:29 +0100 Subject: [PATCH] working on the segmenting --- src/renderer/components/AvatarCreator.tsx | 9 ++- src/renderer/components/Studio.tsx | 12 +-- src/renderer/services/imageService.ts | 99 +++++++++++++++++++++++ 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/src/renderer/components/AvatarCreator.tsx b/src/renderer/components/AvatarCreator.tsx index 4bacb97..a55eb6d 100644 --- a/src/renderer/components/AvatarCreator.tsx +++ b/src/renderer/components/AvatarCreator.tsx @@ -1,5 +1,5 @@ import React, { useState } from 'react'; -import { stitchAssets, fileToDataUrl } from '../services/imageService'; +import { stitchAssets, fileToDataUrl, sliceAvatarSheetAsync } from '../services/imageService'; import { generateAvatarImage } from '../services/geminiService'; import LoadingSpinner from './LoadingSpinner'; @@ -35,7 +35,12 @@ const AvatarCreator: React.FC = ({ onAvatarGenerated }) => { try { const imageUrl = await generateAvatarImage(prompt); - // No automatic analysis - user will rig expressions manually + // Automate extraction of expression parts + const slices = await sliceAvatarSheetAsync(imageUrl); + console.log('Automated slices generated:', slices); + + // We still use the full sheet for the RiggingEditor, + // but the slices are now available for future use (e.g. saving as separate files) onAvatarGenerated(imageUrl, name, {}); } catch (err) { console.error(err); diff --git a/src/renderer/components/Studio.tsx b/src/renderer/components/Studio.tsx index 2894ff4..29f9f54 100644 --- a/src/renderer/components/Studio.tsx +++ b/src/renderer/components/Studio.tsx @@ -182,18 +182,20 @@ const Studio: React.FC = ({ avatar, onBack }) => { const calculateFeaturePosition = (featureRect: Rect, featureType: 'eye' | 'mouth') => { if (!avatar.riggingReference || !featureRect) return { x: 0, y: 0 }; - const { faceCenter, faceWidth, faceHeight } = avatar.riggingReference; + const { faceCenter } = avatar.riggingReference; - // Calculate feature position relative to face center in rigging space + // Calculate feature position relative to face center in rigging space (normalized) const featureCenterX = featureRect.x + featureRect.w / 2; const featureCenterY = featureRect.y + featureRect.h / 2; const relX = featureCenterX - faceCenter.x; const relY = featureCenterY - faceCenter.y; - // Scale relative positions by face width/height to match tracking scale - const scaledX = relX * faceWidth * avatarPosition.scale; - const scaledY = relY * faceHeight * avatarPosition.scale; + // Map normalized 0-1 rigging space to the visual scale of the avatar's "face size" + // Since the container is 600px, and the face usually takes up a good chunk of it, + // multiplying by 600 allows the relative positions to shift the features correctly. + const scaledX = relX * 600 * avatarPosition.scale; + const scaledY = relY * 600 * avatarPosition.scale; return { x: scaledX, diff --git a/src/renderer/services/imageService.ts b/src/renderer/services/imageService.ts index cbc7fb5..c9b83dd 100644 --- a/src/renderer/services/imageService.ts +++ b/src/renderer/services/imageService.ts @@ -19,6 +19,105 @@ export const loadImage = (src: string): Promise => { }); }; +/** + * Extracts a specific rectangle from an image and returns it as a data URL. + */ +export const extractRect = async (image: HTMLImageElement, rect: Rect): Promise => { + const canvas = document.createElement('canvas'); + const x = rect.x * image.width; + const y = rect.y * image.height; + const w = rect.w * image.width; + const h = rect.h * image.height; + + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error("Could not get canvas context"); + + ctx.drawImage(image, x, y, w, h, 0, 0, w, h); + return canvas.toDataURL('image/png'); +}; + +/** + * Automatically slices the AI-generated sprite sheet into separate files based on the EXPRESSION_SYSTEM.md grid. + * + * Grid Layout: + * Row 1: Base Character (Full Width) + * Row 2: Eye Expressions (6 equal cols) + * Row 3: Mouth Expressions (6 equal cols) + */ +export const sliceAvatarSheet = async (imageUrl: string) => { + const image = await loadImage(imageUrl); + const w = image.width; + const h = image.height; + + // Based on the 3-row grid described in EXPRESSION_SYSTEM.md + // We assume equal height for rows, though the base character might be larger. + // In a standard sprite sheet, let's assume rows are roughly divided. + // Row 1 (Base), Row 2 (Eyes), Row 3 (Mouth) + const rowH = h / 3; + const colW = w / 6; + + const slices: Record = {}; + + // 1. Base Face (Full width of Row 1) + slices['base'] = await extractRect(image, { x: 0, y: 0, w: 1, h: 1/3 }); + + // 2. Eye Expressions (Row 2) + const eyeTypes = ['NEUTRAL', 'HAPPY', 'SURPRISED', 'ANGRY', 'SAD', 'BLINK']; + eyeTypes.forEach((type, i) => { + slices[`eye_${type}`] = extractRect(image, { + x: i * (1/6), + y: 1/3, + w: 1/6, + h: 1/3 + }) as any; // In a real loop we should await or Promise.all + }); + + // 3. Mouth Expressions (Row 3) + const mouthTypes = ['NEUTRAL', 'HAPPY', 'OPEN_TALK', 'WIDE_OPEN', 'FROWN', 'O_SHAPE']; + mouthTypes.forEach((type, i) => { + slices[`mouth_${type}`] = extractRect(image, { + x: i * (1/6), + y: 2/3, + w: 1/6, + h: 1/3 + }) as any; + }); + + // Since the above are promises, we need to wrap this properly. + // See implementation below. + return slices; +}; + +// Refined slice function to handle promises properly +export const sliceAvatarSheetAsync = async (imageUrl: string) => { + const image = await loadImage(imageUrl); + const rowH = 1/3; + const colW = 1/6; + + const eyeTypes = ['NEUTRAL', 'HAPPY', 'SURPRISED', 'ANGRY', 'SAD', 'BLINK']; + const mouthTypes = ['NEUTRAL', 'HAPPY', 'OPEN_TALK', 'WIDE_OPEN', 'FROWN', 'O_SHAPE']; + + const results: Record = {}; + + results['base'] = await extractRect(image, { x: 0, y: 0, w: 1, h: rowH }); + + for (let i = 0; i < eyeTypes.length; i++) { + results[`eye_${eyeTypes[i]}`] = await extractRect(image, { + x: i * colW, y: rowH, w: colW, h: rowH + }); + } + + for (let i = 0; i < mouthTypes.length; i++) { + results[`mouth_${mouthTypes[i]}`] = await extractRect(image, { + x: i * colW, y: rowH * 2, w: colW, h: rowH + }); + } + + return results; +}; + export const stitchAssets = async ( baseSrc: string, blinkSrc?: string,