google Mediapipe 2d skeleton to 3d Avatar in Unity

Google Mediapipe Blazepose기반으로 Unity에서 3d position을 이용한 joint의 회전을 계산하는 연구

MediaPipe Unity Plugin을 설치하고 설정해야합니다. 설치 및 설정 방법은 구글의 MediaPipe Unity Plugin 공식 문서에서 확인할 수 있습니다.

다음으로, 아바타 모델을 로드하고, 모델의 어깨 위치를 확인해야합니다. 이는 모델의 적절한 위치에서 해당 정보를 가져오는 방법에 따라 달라질 수 있습니다.

다음으로, MediaPipe Unity Plugin에서 제공하는 "Pose Landmark"를 사용하여 어깨 랜드마크를 확인할 수 있습니다. 이를 통해 어깨의 방향을 계산할 수 있습니다.

아래는 예시 코드입니다.

shoulder

using UnityEngine;
using Mediapipe.Unity;

public class AvatarShoulderOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;
    
    private Vector3 leftShoulderPosition;
    private Vector3 rightShoulderPosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the left and right shoulder positions from the avatar model
        // These positions may need to be adjusted based on the model's size and orientation
        leftShoulderPosition = avatarModel.transform.Find("LeftShoulder").position;
        rightShoulderPosition = avatarModel.transform.Find("RightShoulder").position;

        // Add a pose landmark list to the scene to detect shoulder landmarks
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
        landmarkList.SetLandmarkDetection(RightShoulder);
        landmarkList.SetLandmarkDetection(LeftShoulder);
    }

    void Update()
    {
        // Get the shoulder landmarks from the PoseLandmarkList
        var leftShoulderLandmark = landmarkList.GetLandmark(13); // Left shoulder landmark ID
        var rightShoulderLandmark = landmarkList.GetLandmark(14); // Right shoulder landmark ID

        // Calculate the shoulder direction based on the landmarks
        var leftShoulderDirection = (leftShoulderLandmark.WorldPosition - leftShoulderPosition).normalized;
        var rightShoulderDirection = (rightShoulderLandmark.WorldPosition - rightShoulderPosition).normalized;

        // Set the avatar's shoulder orientation
        avatarModel.transform.Find("LeftShoulder").rotation = Quaternion.LookRotation(leftShoulderDirection, Vector3.up);
        avatarModel.transform.Find("RightShoulder").rotation = Quaternion.LookRotation(rightShoulderDirection, Vector3.up);
    }
}

위의 코드는 아바타 모델의 어깨 방향을 갱신하기 위해 매 프레임마다 PoseLandmarkList를 사용하여 어깨 위치를 확인하고, 방향을 계산하여 아바타의 어깨 방향을 조절합니다.

Hip joint

아래 코드에서는 다음과 같은 작업을 수행합니다.
MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 힙 랜드마크를 식별합니다.
아바타의 힙 조인트 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarHipJointOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;
    private Vector3 hipPosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the hip position from the avatar model
        // This position may need to be adjusted based on the model's size and orientation
        hipPosition = avatarModel.transform.Find("Hip").position;

        // Add a pose landmark list to the scene to detect hip landmark
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
        landmarkList.SetLandmarkDetection(Hip);
    }

    void Update()
    {
        // Get the hip landmark from the PoseLandmarkList
        var hipLandmark = landmarkList.GetLandmark(23); // Hip landmark ID

        // Calculate the hip direction based on the landmark
        var hipDirection = (hipLandmark.WorldPosition - hipPosition).normalized;

        // Set the avatar's hip orientation
        avatarModel.transform.Find("Hip").rotation = Quaternion.LookRotation(hipDirection, Vector3.up);
    }
}

Knee

MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 무릎 랜드마크를 식별합니다.
아바타의 무릎 조인트 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarKneeJointOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;
    private Vector3 kneePosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the knee position from the avatar model
        // This position may need to be adjusted based on the model's size and orientation
        kneePosition = avatarModel.transform.Find("Knee").position;

        // Add a pose landmark list to the scene to detect knee landmark
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
        landmarkList.SetLandmarkDetection(Knee);

        // Reverse the orientation of the model's legs, so that the knee points forward by default
        avatarModel.transform.Find("LeftThigh").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightThigh").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("LeftCalf").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightCalf").localRotation *= Quaternion.Euler(180f, 0f, 0f);
    }

    void Update()
    {
        // Get the knee landmark from the PoseLandmarkList
        var kneeLandmark = landmarkList.GetLandmark(25); // Left knee landmark ID

        // Calculate the knee direction based on the landmark
        var kneeDirection = (kneeLandmark.WorldPosition - kneePosition).normalized;

        // Set the avatar's left knee orientation
        avatarModel.transform.Find("LeftKnee").rotation = Quaternion.LookRotation(kneeDirection, Vector3.up);

        // Set the avatar's right knee orientation
        avatarModel.transform.Find("RightKnee").rotation = Quaternion.LookRotation(kneeDirection, Vector3.up);
    }
}

 

elbow

MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 팔꿈치 랜드마크를 식별합니다.
아바타의 팔꿈치 조인트 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarElbowJointOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;
    private Vector3 elbowPosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the elbow position from the avatar model
        // This position may need to be adjusted based on the model's size and orientation
        elbowPosition = avatarModel.transform.Find("Elbow").position;

        // Add a pose landmark list to the scene to detect elbow landmark
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
        landmarkList.SetLandmarkDetection(Elbow);

        // Reverse the orientation of the model's arms, so that the elbow points forward by default
        avatarModel.transform.Find("LeftArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("LeftForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
    }

    void Update()
    {
        // Get the elbow landmark from the PoseLandmarkList
        var elbowLandmark = landmarkList.GetLandmark(3); // Left elbow landmark ID

        // Calculate the elbow direction based on the landmark
        var elbowDirection = (elbowLandmark.WorldPosition - elbowPosition).normalized;

        // Set the avatar's left elbow orientation
        avatarModel.transform.Find("LeftElbow").rotation = Quaternion.LookRotation(elbowDirection, Vector3.up);

        // Set the avatar's right elbow orientation
        avatarModel.transform.Find("RightElbow").rotation = Quaternion.LookRotation(elbowDirection, Vector3.up);
    }
}

 

wrist

MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 손목 랜드마크를 식별합니다.
아바타의 손목 조인트 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarWristJointOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;
    private Vector3 wristPosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the wrist position from the avatar model
        // This position may need to be adjusted based on the model's size and orientation
        wristPosition = avatarModel.transform.Find("Wrist").position;

        // Add a pose landmark list to the scene to detect wrist landmark
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
        landmarkList.SetLandmarkDetection(Wrist);

        // Reverse the orientation of the model's arms, so that the wrist points forward by default
        avatarModel.transform.Find("LeftArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("LeftForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("LeftElbow").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightElbow").localRotation *= Quaternion.Euler(180f, 0f, 0f);
    }

    void Update()
    {
        // Get the wrist landmark from the PoseLandmarkList
        var wristLandmark = landmarkList.GetLandmark(15); // Left wrist landmark ID

        // Calculate the wrist direction based on the landmark
        var wristDirection = (wristLandmark.WorldPosition - wristPosition).normalized;

        // Set the avatar's left wrist orientation
        avatarModel.transform.Find("LeftHand").rotation = Quaternion.LookRotation(wristDirection, Vector3.up);

        // Set the avatar's right wrist orientation
        avatarModel.transform.Find("RightHand").rotation = Quaternion.LookRotation(wristDirection, Vector3.up);
    }
}

Ankle

MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 발목 랜드마크를 식별합니다.
아바타의 발목 조인트 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarAnkleJointOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;
    private Vector3 anklePosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the ankle position from the avatar model
        // This position may need to be adjusted based on the model's size and orientation
        anklePosition = avatarModel.transform.Find("Ankle").position;

        // Add a pose landmark list to the scene to detect ankle landmark
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
        landmarkList.SetLandmarkDetection(Ankle);

        // Reverse the orientation of the model's legs, so that the ankle points forward by default
        avatarModel.transform.Find("LeftUpLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightUpLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("LeftLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("LeftFoot").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightFoot").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("LeftToeBase").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightToeBase").localRotation *= Quaternion.Euler(180f, 0f, 0f);
    }

    void Update()
    {
        // Get the ankle landmark from the PoseLandmarkList
        var ankleLandmark = landmarkList.GetLandmark(16); // Left ankle landmark ID

        // Calculate the ankle direction based on the landmark
        var ankleDirection = (ankleLandmark.WorldPosition - anklePosition).normalized;

        // Set the avatar's left ankle orientation
        avatarModel.transform.Find("LeftFoot").rotation = Quaternion.LookRotation(ankleDirection, Vector3.up);

        // Set the avatar's right ankle orientation
        avatarModel.transform.Find("RightFoot").rotation = Quaternion.LookRotation(ankleDirection, Vector3.up);
    }
}

Hand

MediaPipe Unity Plugin의 Hand Landmark Detection 기능을 사용하여 아바타의 손 랜드마크를 식별합니다.
아바타의 손 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarHandOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private HandLandmarkList landmarkList;
    private Vector3 wristPosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the wrist position from the avatar model
        // This position may need to be adjusted based on the model's size and orientation
        wristPosition = avatarModel.transform.Find("RightHand/Wrist").position;

        // Add a hand landmark list to the scene to detect hand landmarks
        landmarkList = gameObject.AddComponent<HandLandmarkList>();
        landmarkList.enableDetection = true;

        // Reverse the orientation of the model's right arm, so that the hand points forward by default
        avatarModel.transform.Find("RightArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightForeArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightHand").localRotation *= Quaternion.Euler(180f, 0f, 0f);
    }

    void Update()
    {
        // Get the wrist landmark from the HandLandmarkList
        var wristLandmark = landmarkList.GetLandmark(0); // Right wrist landmark ID

        // Calculate the hand direction based on the landmarks
        var handDirection = (landmarkList.GetLandmark(5).WorldPosition - landmarkList.GetLandmark(17).WorldPosition).normalized;

        // Set the avatar's right hand orientation
        avatarModel.transform.Find("RightHand").rotation = Quaternion.LookRotation(handDirection, Vector3.up);

        // Set the avatar's right forearm orientation
        var forearmDirection = (wristLandmark.WorldPosition - wristPosition).normalized;
        avatarModel.transform.Find("RightForeArm").rotation = Quaternion.LookRotation(forearmDirection, -handDirection);

        // Set the avatar's right arm orientation
        var armDirection = (wristPosition - avatarModel.transform.Find("RightShoulder").position).normalized;
        avatarModel.transform.Find("RightArm").rotation = Quaternion.LookRotation(armDirection, -forearmDirection);
    }
}

Foot

MediaPipe Unity Plugin의 Feet Landmark Detection 기능을 사용하여 아바타의 발 랜드마크를 식별합니다.
아바타의 발 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarFootOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private FeetLandmarkList landmarkList;
    private Vector3 anklePosition;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Get the ankle position from the avatar model
        // This position may need to be adjusted based on the model's size and orientation
        anklePosition = avatarModel.transform.Find("RightLeg/RightFoot/Ankle").position;

        // Add a feet landmark list to the scene to detect feet landmarks
        landmarkList = gameObject.AddComponent<FeetLandmarkList>();
        landmarkList.enableDetection = true;

        // Reverse the orientation of the model's right leg, so that the foot points forward by default
        avatarModel.transform.Find("RightLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
        avatarModel.transform.Find("RightFoot").localRotation *= Quaternion.Euler(180f, 0f, 0f);
    }

    void Update()
    {
        // Get the ankle landmark from the FeetLandmarkList
        var ankleLandmark = landmarkList.GetLandmark(0); // Right ankle landmark ID

        // Calculate the foot direction based on the landmarks
        var footDirection = (landmarkList.GetLandmark(8).WorldPosition - landmarkList.GetLandmark(17).WorldPosition).normalized;

        // Set the avatar's right foot orientation
        avatarModel.transform.Find("RightFoot").rotation = Quaternion.LookRotation(footDirection, Vector3.up);

        // Set the avatar's right leg orientation
        var legDirection = (anklePosition - avatarModel.transform.Find("RightHip").position).normalized;
        avatarModel.transform.Find("RightLeg").rotation = Quaternion.LookRotation(legDirection, -footDirection);
    }
}

LeftHip

MediaPipe Unity Plugin의 Pose Detection 기능을 사용하여 아바타의 랜드마크를 식별합니다.
아바타의 왼쪽 엉덩이 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarLeftHipOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Add a pose landmark list to the scene to detect pose landmarks
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
    }

    void Update()
    {
        // Get the left hip landmark from the PoseLandmarkList
        var leftHipLandmark = landmarkList.GetLandmark((int)PoseLandmark.LeftHip);

        // Get the left knee landmark from the PoseLandmarkList
        var leftKneeLandmark = landmarkList.GetLandmark((int)PoseLandmark.LeftKnee);

        // Get the left ankle landmark from the PoseLandmarkList
        var leftAnkleLandmark = landmarkList.GetLandmark((int)PoseLandmark.LeftAnkle);

        // Calculate the leg direction based on the landmarks
        var legDirection = (leftAnkleLandmark.WorldPosition - leftHipLandmark.WorldPosition).normalized;

        // Calculate the foot direction based on the landmarks
        var footDirection = (leftAnkleLandmark.WorldPosition - leftKneeLandmark.WorldPosition).normalized;

        // Set the avatar's left leg orientation
        avatarModel.transform.Find("LeftLeg").rotation = Quaternion.LookRotation(legDirection, -footDirection);
    }
}

RightHip

MediaPipe Unity Plugin의 Pose Detection 기능을 사용하여 아바타의 랜드마크를 식별합니다.
아바타의 오른쪽 엉덩이 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity;

public class AvatarRightHipOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private PoseLandmarkList landmarkList;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Add a pose landmark list to the scene to detect pose landmarks
        landmarkList = gameObject.AddComponent<PoseLandmarkList>();
        landmarkList.enableDetection = true;
    }

    void Update()
    {
        // Get the right hip landmark from the PoseLandmarkList
        var rightHipLandmark = landmarkList.GetLandmark((int)PoseLandmark.RightHip);

        // Get the right knee landmark from the PoseLandmarkList
        var rightKneeLandmark = landmarkList.GetLandmark((int)PoseLandmark.RightKnee);

        // Get the right ankle landmark from the PoseLandmarkList
        var rightAnkleLandmark = landmarkList.GetLandmark((int)PoseLandmark.RightAnkle);

        // Calculate the leg direction based on the landmarks
        var legDirection = (rightAnkleLandmark.WorldPosition - rightHipLandmark.WorldPosition).normalized;

        // Calculate the foot direction based on the landmarks
        var footDirection = (rightAnkleLandmark.WorldPosition - rightKneeLandmark.WorldPosition).normalized;

        // Set the avatar's right leg orientation
        avatarModel.transform.Find("RightLeg").rotation = Quaternion.LookRotation(legDirection, -footDirection);
    }
}

손가락

MediaPipe Unity Plugin의 Hand Tracking 기능을 사용하여 아바타의 손가락 랜드마크를 식별합니다.
각 손가락 방향을 계산하고, 방향을 조절합니다.

using UnityEngine;
using Mediapipe.Unity.HandTracking;

public class AvatarFingerOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private HandTrackingLandmarkList landmarkList;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Add a hand landmark list to the scene to detect hand landmarks
        landmarkList = gameObject.AddComponent<HandTrackingLandmarkList>();
        landmarkList.enableDetection = true;
    }

    void Update()
    {
        // Get the finger landmarks from the HandTrackingLandmarkList
        var thumbLandmark = landmarkList.GetLandmark(HandLandmark.ThumbTip);
        var indexLandmark = landmarkList.GetLandmark(HandLandmark.IndexTip);
        var middleLandmark = landmarkList.GetLandmark(HandLandmark.MiddleTip);
        var ringLandmark = landmarkList.GetLandmark(HandLandmark.RingTip);
        var pinkyLandmark = landmarkList.GetLandmark(HandLandmark.PinkyTip);

        // Calculate the direction of each finger based on the landmarks
        var thumbDirection = (thumbLandmark.WorldPosition - indexLandmark.WorldPosition).normalized;
        var indexDirection = (indexLandmark.WorldPosition - middleLandmark.WorldPosition).normalized;
        var middleDirection = (middleLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;
        var ringDirection = (ringLandmark.WorldPosition - pinkyLandmark.WorldPosition).normalized;
        var pinkyDirection = (pinkyLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;

        // Set the avatar's finger orientations
        avatarModel.transform.Find("RightHand/Thumb").rotation = Quaternion.LookRotation(thumbDirection);
        avatarModel.transform.Find("RightHand/Index").rotation = Quaternion.LookRotation(indexDirection);
        avatarModel.transform.Find("RightHand/Middle").rotation = Quaternion.LookRotation(middleDirection);
        avatarModel.transform.Find("RightHand/Ring").rotation = Quaternion.LookRotation(ringDirection);
        avatarModel.transform.Find("RightHand/Pinky").rotation = Quaternion.LookRotation(pinkyDirection);
    }
}

오른손과 손가락

이전에 작성한 코드에서 RightHand만 바꾸면 왼손과 오른손 모두를 지원할 수 있습니다. 아래는 MediaPipe Unity Plugin을 사용하여 3D 아바타의 오른손의 손가락 방향을 계산하는 샘플 코드입니다.

using UnityEngine;
using Mediapipe.Unity.HandTracking;

public class AvatarRightHandFingerOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private HandTrackingLandmarkList landmarkList;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Add a hand landmark list to the scene to detect hand landmarks
        landmarkList = gameObject.AddComponent<HandTrackingLandmarkList>();
        landmarkList.enableDetection = true;
    }

    void Update()
    {
        // Get the finger landmarks from the HandTrackingLandmarkList
        var thumbLandmark = landmarkList.GetLandmark(HandLandmark.ThumbTip);
        var indexLandmark = landmarkList.GetLandmark(HandLandmark.IndexTip);
        var middleLandmark = landmarkList.GetLandmark(HandLandmark.MiddleTip);
        var ringLandmark = landmarkList.GetLandmark(HandLandmark.RingTip);
        var pinkyLandmark = landmarkList.GetLandmark(HandLandmark.PinkyTip);

        // Calculate the direction of each finger based on the landmarks
        var thumbDirection = (thumbLandmark.WorldPosition - indexLandmark.WorldPosition).normalized;
        var indexDirection = (indexLandmark.WorldPosition - middleLandmark.WorldPosition).normalized;
        var middleDirection = (middleLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;
        var ringDirection = (ringLandmark.WorldPosition - pinkyLandmark.WorldPosition).normalized;
        var pinkyDirection = (pinkyLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;

        // Set the avatar's finger orientations
        avatarModel.transform.Find("RightHand/Thumb").rotation = Quaternion.LookRotation(thumbDirection);
        avatarModel.transform.Find("RightHand/Index").rotation = Quaternion.LookRotation(indexDirection);
        avatarModel.transform.Find("RightHand/Middle").rotation = Quaternion.LookRotation(middleDirection);
        avatarModel.transform.Find("RightHand/Ring").rotation = Quaternion.LookRotation(ringDirection);
        avatarModel.transform.Find("RightHand/Pinky").rotation = Quaternion.LookRotation(pinkyDirection);
    }
}

머리와 목

이전에 작성한 코드에서 Head와 Neck만 추가하면 머리와 목의 방향도 계산할 수 있습니다. 아래는 MediaPipe Unity Plugin을 사용하여 3D 아바타의 머리와 목의 방향을 계산하는 샘플 코드입니다.

using UnityEngine;
using Mediapipe.Unity.FaceDetection;

public class AvatarHeadNeckOrientation : MonoBehaviour
{
    [SerializeField] private GameObject avatarModel;
    [SerializeField] private FaceDetectionSolution faceDetection;

    void Start()
    {
        // Load the avatar model
        // You may need to adjust the position and rotation to fit the model in the scene
        Instantiate(avatarModel, Vector3.zero, Quaternion.identity);

        // Add a FaceDetectionSolution to the scene to detect the face
        faceDetection = gameObject.AddComponent<FaceDetectionSolution>();
        faceDetection.enableFaceDetection = true;
    }

    void Update()
    {
        // Get the head and neck landmarks from the FaceDetectionSolution
        var headLandmark = faceDetection.faceLandmarkList?.GetLandmark(FaceLandmark.HeadTop);
        var neckLandmark = faceDetection.faceLandmarkList?.GetLandmark(FaceLandmark.NoseTip);

        // Calculate the direction of the head based on the landmarks
        var headDirection = (headLandmark?.WorldPosition - neckLandmark?.WorldPosition)?.normalized ?? Vector3.forward;

        // Calculate the direction of the neck based on the landmarks
        var neckDirection = (neckLandmark?.WorldPosition - Vector3.up)?.normalized ?? Vector3.up;

        // Set the avatar's head and neck orientations
        avatarModel.transform.Find("Head").rotation = Quaternion.LookRotation(headDirection, neckDirection);
        avatarModel.transform.Find("Neck").rotation = Quaternion.LookRotation(neckDirection);
    }
}