Blazepose 3d Avatar Simple

Mediapipe의 unity version으로 Blazepose가 있습니다. https://github.com/homuler/MediaPipeUnityPlugin

간단하게 3D Avatar에게 적용해 본코드 입니다.

Avatar는 https://www.mixamo.com/  에서 T포즈를 하고 있는 x bot.fbx를 다운받아서 사용했습니다.

using System.Collections;
using System.Collections.Generic;
using Mediapipe;
using Mediapipe.Unity;
using UnityEngine;

public class Test : MonoBehaviour
{
    public Transform LeftForearm;
    public Transform LeftShoulder;
    public Transform RightForearm;
    public Transform RightShoulder;
    public Transform LeftHip;
    public Transform LeftKnee;
    public Transform RightHip;
    public Transform RightKnee;

    public TMPro.TextMeshProUGUI text;

    public PoseWorldLandmarkListAnnotationController controller;

    private static readonly int NOSE = 0;
    private static readonly int L_EYE_INNER = 1;
    private static readonly int L_EYE = 2;
    private static readonly int L_EYE_OUTER = 3;
    private static readonly int R_EYE_INNER = 4;
    private static readonly int R_EYE = 5;
    private static readonly int R_EYE_OUTER = 6;
    private static readonly int L_EAR = 7;
    private static readonly int R_EAR = 8;
    private static readonly int L_MOUTH = 9;
    private static readonly int R_MOUTH = 10;
    private static readonly int L_SHOULDER = 12;
    private static readonly int R_SHOULDER = 11;
    private static readonly int L_ELBOW = 14;
    private static readonly int R_ELBOW = 13;
    private static readonly int L_WRIST = 16;
    private static readonly int R_WRIST = 15;
    private static readonly int L_PINKY = 17;
    private static readonly int R_PINKY = 18;
    private static readonly int L_INDEX = 19;
    private static readonly int R_INDEX = 20;
    private static readonly int L_THUMB = 21;
    private static readonly int R_THUMB = 22;
    private static readonly int L_HIP = 24;
    private static readonly int R_HIP = 23;
    private static readonly int L_KNEE = 26;
    private static readonly int R_KNEE = 25;
    private static readonly int L_ANKLE = 28;
    private static readonly int R_ANKLE = 27;
    private static readonly int L_HEEL = 30;
    private static readonly int R_HEEL = 29;
    private static readonly int L_FINDEX = 32;
    private static readonly int R_FINDEX = 31;

    [SerializeField] private Animator animator;
    void Start()
    {
        LeftForearm = animator.GetBoneTransform(HumanBodyBones.LeftLowerArm);       //elbow
        LeftShoulder = animator.GetBoneTransform(HumanBodyBones.LeftUpperArm);      //shoulder
        RightForearm = animator.GetBoneTransform(HumanBodyBones.RightLowerArm);     //elbow
        RightShoulder = animator.GetBoneTransform(HumanBodyBones.RightUpperArm);    //shoulder
        LeftHip = animator.GetBoneTransform(HumanBodyBones.LeftUpperLeg);
        LeftKnee = animator.GetBoneTransform(HumanBodyBones.LeftLowerLeg);
        RightHip = animator.GetBoneTransform(HumanBodyBones.RightUpperLeg);
        RightKnee = animator.GetBoneTransform(HumanBodyBones.RightLowerLeg);

        text.text = "";

        controller.PoseLandmarkEvent.AddListener((v) =>
        {
            if (v.Count!=null)
            {
                text.text = "";
                for (int i=0; i<v.Count; i++)
                {
                    var landmark = v[i];
                    text.text += $"{i} : {landmark}\n";
                }

                UpdateJoint(v);
            }
        });
    }

    static Vector3 ToVector3(Landmark landmark)
    {
        return new Vector3(landmark.X, landmark.Y, landmark.Z);
    }

    static Vector3 resolution = new Vector3(640, 480, -320); 
    static Vector3 ToVector3(NormalizedLandmark landmark)
    {
        return new Vector3(landmark.X * resolution.x, (1f-landmark.Y) * resolution.y, landmark.Z * resolution.z);
    }

    // Update is called once per frame
    void UpdateJoint(IList<Landmark> landmarks)
    {
        if (landmarks.Count <= 0)
            return;
        

        // upper arms
        Vector3 wristLeft    = ToVector3(landmarks[L_WRIST]);
        Vector3 elbowLeft    = ToVector3(landmarks[L_ELBOW]);
        Vector3 shoulderLeft = ToVector3(landmarks[L_SHOULDER]);
        this.LeftForearm.rotation  = Quaternion.Slerp(this.LeftForearm.rotation, Quaternion.FromToRotation(Vector3.up, elbowLeft - wristLeft), 10f * Time.deltaTime);
        this.LeftShoulder.rotation = Quaternion.Slerp(this.LeftShoulder.rotation, Quaternion.FromToRotation(Vector3.up, shoulderLeft - elbowLeft), 10f * Time.deltaTime);
        
        Vector3 wristRight    = ToVector3(landmarks[R_WRIST]);
        Vector3 elbowRight    = ToVector3(landmarks[R_ELBOW]);
        Vector3 shoulderRight = ToVector3(landmarks[R_SHOULDER]);
        this.RightForearm.rotation  = Quaternion.Slerp(this.RightForearm.rotation, Quaternion.FromToRotation(Vector3.up, elbowRight - wristRight), 10f * Time.deltaTime);
        this.RightShoulder.rotation = Quaternion.Slerp(this.RightShoulder.rotation, Quaternion.FromToRotation(Vector3.up, shoulderRight - elbowRight), 10f * Time.deltaTime);

        // lower legs
        Vector3 hipLeft   = ToVector3(landmarks[L_HIP]);     
        Vector3 kneeLeft  = ToVector3(landmarks[L_KNEE]);   
        Vector3 ankleLeft = ToVector3(landmarks[L_ANKLE]); 
        Vector3 heelLeft  = ToVector3(landmarks[L_HEEL]);   
        this.LeftKnee.rotation = Quaternion.Slerp(this.LeftKnee.rotation, Quaternion.FromToRotation(Vector3.up, kneeLeft - ankleLeft), 10f * Time.deltaTime);
        this.LeftHip.rotation  = Quaternion.Slerp(this.LeftHip.rotation, Quaternion.FromToRotation(Vector3.up, hipLeft - kneeLeft), 10f * Time.deltaTime);

        Vector3 hipRight   = ToVector3(landmarks[R_HIP]);
        Vector3 kneeRight  = ToVector3(landmarks[R_KNEE]);
        Vector3 ankleRight = ToVector3(landmarks[R_ANKLE]);
        Vector3 heelRight  = ToVector3(landmarks[R_HEEL]);
        this.RightKnee.rotation = Quaternion.Slerp(this.RightKnee.rotation, Quaternion.FromToRotation(Vector3.up, kneeRight - ankleRight), 10f * Time.deltaTime);
        this.RightHip.rotation  = Quaternion.Slerp(this.RightHip.rotation, Quaternion.FromToRotation(Vector3.up, hipRight - kneeRight), 10f * Time.deltaTime);

    }
}

간단한 만큼 동작의 디지털 트윈이 정교하지 않습니다.
다음버전에선 다른 정보들을 추가 이용하여 이를 개선해 봅니다.