Google Mediapipe Blazepose기반으로 Unity에서 3d position을 이용한 joint의 회전을 계산하는 연구
MediaPipe Unity Plugin을 설치하고 설정해야합니다. 설치 및 설정 방법은 구글의 MediaPipe Unity Plugin 공식 문서에서 확인할 수 있습니다.
다음으로, 아바타 모델을 로드하고, 모델의 어깨 위치를 확인해야합니다. 이는 모델의 적절한 위치에서 해당 정보를 가져오는 방법에 따라 달라질 수 있습니다.
다음으로, MediaPipe Unity Plugin에서 제공하는 "Pose Landmark"를 사용하여 어깨 랜드마크를 확인할 수 있습니다. 이를 통해 어깨의 방향을 계산할 수 있습니다.
아래는 예시 코드입니다.
shoulder
using UnityEngine;
using Mediapipe.Unity;
public class AvatarShoulderOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
private Vector3 leftShoulderPosition;
private Vector3 rightShoulderPosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the left and right shoulder positions from the avatar model
// These positions may need to be adjusted based on the model's size and orientation
leftShoulderPosition = avatarModel.transform.Find("LeftShoulder").position;
rightShoulderPosition = avatarModel.transform.Find("RightShoulder").position;
// Add a pose landmark list to the scene to detect shoulder landmarks
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
landmarkList.SetLandmarkDetection(RightShoulder);
landmarkList.SetLandmarkDetection(LeftShoulder);
}
void Update()
{
// Get the shoulder landmarks from the PoseLandmarkList
var leftShoulderLandmark = landmarkList.GetLandmark(13); // Left shoulder landmark ID
var rightShoulderLandmark = landmarkList.GetLandmark(14); // Right shoulder landmark ID
// Calculate the shoulder direction based on the landmarks
var leftShoulderDirection = (leftShoulderLandmark.WorldPosition - leftShoulderPosition).normalized;
var rightShoulderDirection = (rightShoulderLandmark.WorldPosition - rightShoulderPosition).normalized;
// Set the avatar's shoulder orientation
avatarModel.transform.Find("LeftShoulder").rotation = Quaternion.LookRotation(leftShoulderDirection, Vector3.up);
avatarModel.transform.Find("RightShoulder").rotation = Quaternion.LookRotation(rightShoulderDirection, Vector3.up);
}
}
위의 코드는 아바타 모델의 어깨 방향을 갱신하기 위해 매 프레임마다 PoseLandmarkList를 사용하여 어깨 위치를 확인하고, 방향을 계산하여 아바타의 어깨 방향을 조절합니다.
Hip joint
아래 코드에서는 다음과 같은 작업을 수행합니다.
MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 힙 랜드마크를 식별합니다.
아바타의 힙 조인트 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarHipJointOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
private Vector3 hipPosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the hip position from the avatar model
// This position may need to be adjusted based on the model's size and orientation
hipPosition = avatarModel.transform.Find("Hip").position;
// Add a pose landmark list to the scene to detect hip landmark
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
landmarkList.SetLandmarkDetection(Hip);
}
void Update()
{
// Get the hip landmark from the PoseLandmarkList
var hipLandmark = landmarkList.GetLandmark(23); // Hip landmark ID
// Calculate the hip direction based on the landmark
var hipDirection = (hipLandmark.WorldPosition - hipPosition).normalized;
// Set the avatar's hip orientation
avatarModel.transform.Find("Hip").rotation = Quaternion.LookRotation(hipDirection, Vector3.up);
}
}
Knee
MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 무릎 랜드마크를 식별합니다.
아바타의 무릎 조인트 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarKneeJointOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
private Vector3 kneePosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the knee position from the avatar model
// This position may need to be adjusted based on the model's size and orientation
kneePosition = avatarModel.transform.Find("Knee").position;
// Add a pose landmark list to the scene to detect knee landmark
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
landmarkList.SetLandmarkDetection(Knee);
// Reverse the orientation of the model's legs, so that the knee points forward by default
avatarModel.transform.Find("LeftThigh").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightThigh").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("LeftCalf").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightCalf").localRotation *= Quaternion.Euler(180f, 0f, 0f);
}
void Update()
{
// Get the knee landmark from the PoseLandmarkList
var kneeLandmark = landmarkList.GetLandmark(25); // Left knee landmark ID
// Calculate the knee direction based on the landmark
var kneeDirection = (kneeLandmark.WorldPosition - kneePosition).normalized;
// Set the avatar's left knee orientation
avatarModel.transform.Find("LeftKnee").rotation = Quaternion.LookRotation(kneeDirection, Vector3.up);
// Set the avatar's right knee orientation
avatarModel.transform.Find("RightKnee").rotation = Quaternion.LookRotation(kneeDirection, Vector3.up);
}
}
elbow
MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 팔꿈치 랜드마크를 식별합니다.
아바타의 팔꿈치 조인트 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarElbowJointOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
private Vector3 elbowPosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the elbow position from the avatar model
// This position may need to be adjusted based on the model's size and orientation
elbowPosition = avatarModel.transform.Find("Elbow").position;
// Add a pose landmark list to the scene to detect elbow landmark
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
landmarkList.SetLandmarkDetection(Elbow);
// Reverse the orientation of the model's arms, so that the elbow points forward by default
avatarModel.transform.Find("LeftArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("LeftForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
}
void Update()
{
// Get the elbow landmark from the PoseLandmarkList
var elbowLandmark = landmarkList.GetLandmark(3); // Left elbow landmark ID
// Calculate the elbow direction based on the landmark
var elbowDirection = (elbowLandmark.WorldPosition - elbowPosition).normalized;
// Set the avatar's left elbow orientation
avatarModel.transform.Find("LeftElbow").rotation = Quaternion.LookRotation(elbowDirection, Vector3.up);
// Set the avatar's right elbow orientation
avatarModel.transform.Find("RightElbow").rotation = Quaternion.LookRotation(elbowDirection, Vector3.up);
}
}
wrist
MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 손목 랜드마크를 식별합니다.
아바타의 손목 조인트 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarWristJointOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
private Vector3 wristPosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the wrist position from the avatar model
// This position may need to be adjusted based on the model's size and orientation
wristPosition = avatarModel.transform.Find("Wrist").position;
// Add a pose landmark list to the scene to detect wrist landmark
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
landmarkList.SetLandmarkDetection(Wrist);
// Reverse the orientation of the model's arms, so that the wrist points forward by default
avatarModel.transform.Find("LeftArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("LeftForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightForearm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("LeftElbow").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightElbow").localRotation *= Quaternion.Euler(180f, 0f, 0f);
}
void Update()
{
// Get the wrist landmark from the PoseLandmarkList
var wristLandmark = landmarkList.GetLandmark(15); // Left wrist landmark ID
// Calculate the wrist direction based on the landmark
var wristDirection = (wristLandmark.WorldPosition - wristPosition).normalized;
// Set the avatar's left wrist orientation
avatarModel.transform.Find("LeftHand").rotation = Quaternion.LookRotation(wristDirection, Vector3.up);
// Set the avatar's right wrist orientation
avatarModel.transform.Find("RightHand").rotation = Quaternion.LookRotation(wristDirection, Vector3.up);
}
}
Ankle
MediaPipe Unity Plugin의 Pose Landmark Detection 기능을 사용하여 아바타의 발목 랜드마크를 식별합니다.
아바타의 발목 조인트 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarAnkleJointOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
private Vector3 anklePosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the ankle position from the avatar model
// This position may need to be adjusted based on the model's size and orientation
anklePosition = avatarModel.transform.Find("Ankle").position;
// Add a pose landmark list to the scene to detect ankle landmark
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
landmarkList.SetLandmarkDetection(Ankle);
// Reverse the orientation of the model's legs, so that the ankle points forward by default
avatarModel.transform.Find("LeftUpLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightUpLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("LeftLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("LeftFoot").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightFoot").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("LeftToeBase").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightToeBase").localRotation *= Quaternion.Euler(180f, 0f, 0f);
}
void Update()
{
// Get the ankle landmark from the PoseLandmarkList
var ankleLandmark = landmarkList.GetLandmark(16); // Left ankle landmark ID
// Calculate the ankle direction based on the landmark
var ankleDirection = (ankleLandmark.WorldPosition - anklePosition).normalized;
// Set the avatar's left ankle orientation
avatarModel.transform.Find("LeftFoot").rotation = Quaternion.LookRotation(ankleDirection, Vector3.up);
// Set the avatar's right ankle orientation
avatarModel.transform.Find("RightFoot").rotation = Quaternion.LookRotation(ankleDirection, Vector3.up);
}
}
Hand
MediaPipe Unity Plugin의 Hand Landmark Detection 기능을 사용하여 아바타의 손 랜드마크를 식별합니다.
아바타의 손 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarHandOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private HandLandmarkList landmarkList;
private Vector3 wristPosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the wrist position from the avatar model
// This position may need to be adjusted based on the model's size and orientation
wristPosition = avatarModel.transform.Find("RightHand/Wrist").position;
// Add a hand landmark list to the scene to detect hand landmarks
landmarkList = gameObject.AddComponent<HandLandmarkList>();
landmarkList.enableDetection = true;
// Reverse the orientation of the model's right arm, so that the hand points forward by default
avatarModel.transform.Find("RightArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightForeArm").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightHand").localRotation *= Quaternion.Euler(180f, 0f, 0f);
}
void Update()
{
// Get the wrist landmark from the HandLandmarkList
var wristLandmark = landmarkList.GetLandmark(0); // Right wrist landmark ID
// Calculate the hand direction based on the landmarks
var handDirection = (landmarkList.GetLandmark(5).WorldPosition - landmarkList.GetLandmark(17).WorldPosition).normalized;
// Set the avatar's right hand orientation
avatarModel.transform.Find("RightHand").rotation = Quaternion.LookRotation(handDirection, Vector3.up);
// Set the avatar's right forearm orientation
var forearmDirection = (wristLandmark.WorldPosition - wristPosition).normalized;
avatarModel.transform.Find("RightForeArm").rotation = Quaternion.LookRotation(forearmDirection, -handDirection);
// Set the avatar's right arm orientation
var armDirection = (wristPosition - avatarModel.transform.Find("RightShoulder").position).normalized;
avatarModel.transform.Find("RightArm").rotation = Quaternion.LookRotation(armDirection, -forearmDirection);
}
}
Foot
MediaPipe Unity Plugin의 Feet Landmark Detection 기능을 사용하여 아바타의 발 랜드마크를 식별합니다.
아바타의 발 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarFootOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private FeetLandmarkList landmarkList;
private Vector3 anklePosition;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Get the ankle position from the avatar model
// This position may need to be adjusted based on the model's size and orientation
anklePosition = avatarModel.transform.Find("RightLeg/RightFoot/Ankle").position;
// Add a feet landmark list to the scene to detect feet landmarks
landmarkList = gameObject.AddComponent<FeetLandmarkList>();
landmarkList.enableDetection = true;
// Reverse the orientation of the model's right leg, so that the foot points forward by default
avatarModel.transform.Find("RightLeg").localRotation *= Quaternion.Euler(180f, 0f, 0f);
avatarModel.transform.Find("RightFoot").localRotation *= Quaternion.Euler(180f, 0f, 0f);
}
void Update()
{
// Get the ankle landmark from the FeetLandmarkList
var ankleLandmark = landmarkList.GetLandmark(0); // Right ankle landmark ID
// Calculate the foot direction based on the landmarks
var footDirection = (landmarkList.GetLandmark(8).WorldPosition - landmarkList.GetLandmark(17).WorldPosition).normalized;
// Set the avatar's right foot orientation
avatarModel.transform.Find("RightFoot").rotation = Quaternion.LookRotation(footDirection, Vector3.up);
// Set the avatar's right leg orientation
var legDirection = (anklePosition - avatarModel.transform.Find("RightHip").position).normalized;
avatarModel.transform.Find("RightLeg").rotation = Quaternion.LookRotation(legDirection, -footDirection);
}
}
LeftHip
MediaPipe Unity Plugin의 Pose Detection 기능을 사용하여 아바타의 랜드마크를 식별합니다.
아바타의 왼쪽 엉덩이 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarLeftHipOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Add a pose landmark list to the scene to detect pose landmarks
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
}
void Update()
{
// Get the left hip landmark from the PoseLandmarkList
var leftHipLandmark = landmarkList.GetLandmark((int)PoseLandmark.LeftHip);
// Get the left knee landmark from the PoseLandmarkList
var leftKneeLandmark = landmarkList.GetLandmark((int)PoseLandmark.LeftKnee);
// Get the left ankle landmark from the PoseLandmarkList
var leftAnkleLandmark = landmarkList.GetLandmark((int)PoseLandmark.LeftAnkle);
// Calculate the leg direction based on the landmarks
var legDirection = (leftAnkleLandmark.WorldPosition - leftHipLandmark.WorldPosition).normalized;
// Calculate the foot direction based on the landmarks
var footDirection = (leftAnkleLandmark.WorldPosition - leftKneeLandmark.WorldPosition).normalized;
// Set the avatar's left leg orientation
avatarModel.transform.Find("LeftLeg").rotation = Quaternion.LookRotation(legDirection, -footDirection);
}
}
RightHip
MediaPipe Unity Plugin의 Pose Detection 기능을 사용하여 아바타의 랜드마크를 식별합니다.
아바타의 오른쪽 엉덩이 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity;
public class AvatarRightHipOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private PoseLandmarkList landmarkList;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Add a pose landmark list to the scene to detect pose landmarks
landmarkList = gameObject.AddComponent<PoseLandmarkList>();
landmarkList.enableDetection = true;
}
void Update()
{
// Get the right hip landmark from the PoseLandmarkList
var rightHipLandmark = landmarkList.GetLandmark((int)PoseLandmark.RightHip);
// Get the right knee landmark from the PoseLandmarkList
var rightKneeLandmark = landmarkList.GetLandmark((int)PoseLandmark.RightKnee);
// Get the right ankle landmark from the PoseLandmarkList
var rightAnkleLandmark = landmarkList.GetLandmark((int)PoseLandmark.RightAnkle);
// Calculate the leg direction based on the landmarks
var legDirection = (rightAnkleLandmark.WorldPosition - rightHipLandmark.WorldPosition).normalized;
// Calculate the foot direction based on the landmarks
var footDirection = (rightAnkleLandmark.WorldPosition - rightKneeLandmark.WorldPosition).normalized;
// Set the avatar's right leg orientation
avatarModel.transform.Find("RightLeg").rotation = Quaternion.LookRotation(legDirection, -footDirection);
}
}
손가락
MediaPipe Unity Plugin의 Hand Tracking 기능을 사용하여 아바타의 손가락 랜드마크를 식별합니다.
각 손가락 방향을 계산하고, 방향을 조절합니다.
using UnityEngine;
using Mediapipe.Unity.HandTracking;
public class AvatarFingerOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private HandTrackingLandmarkList landmarkList;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Add a hand landmark list to the scene to detect hand landmarks
landmarkList = gameObject.AddComponent<HandTrackingLandmarkList>();
landmarkList.enableDetection = true;
}
void Update()
{
// Get the finger landmarks from the HandTrackingLandmarkList
var thumbLandmark = landmarkList.GetLandmark(HandLandmark.ThumbTip);
var indexLandmark = landmarkList.GetLandmark(HandLandmark.IndexTip);
var middleLandmark = landmarkList.GetLandmark(HandLandmark.MiddleTip);
var ringLandmark = landmarkList.GetLandmark(HandLandmark.RingTip);
var pinkyLandmark = landmarkList.GetLandmark(HandLandmark.PinkyTip);
// Calculate the direction of each finger based on the landmarks
var thumbDirection = (thumbLandmark.WorldPosition - indexLandmark.WorldPosition).normalized;
var indexDirection = (indexLandmark.WorldPosition - middleLandmark.WorldPosition).normalized;
var middleDirection = (middleLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;
var ringDirection = (ringLandmark.WorldPosition - pinkyLandmark.WorldPosition).normalized;
var pinkyDirection = (pinkyLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;
// Set the avatar's finger orientations
avatarModel.transform.Find("RightHand/Thumb").rotation = Quaternion.LookRotation(thumbDirection);
avatarModel.transform.Find("RightHand/Index").rotation = Quaternion.LookRotation(indexDirection);
avatarModel.transform.Find("RightHand/Middle").rotation = Quaternion.LookRotation(middleDirection);
avatarModel.transform.Find("RightHand/Ring").rotation = Quaternion.LookRotation(ringDirection);
avatarModel.transform.Find("RightHand/Pinky").rotation = Quaternion.LookRotation(pinkyDirection);
}
}
오른손과 손가락
이전에 작성한 코드에서 RightHand만 바꾸면 왼손과 오른손 모두를 지원할 수 있습니다. 아래는 MediaPipe Unity Plugin을 사용하여 3D 아바타의 오른손의 손가락 방향을 계산하는 샘플 코드입니다.
using UnityEngine;
using Mediapipe.Unity.HandTracking;
public class AvatarRightHandFingerOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private HandTrackingLandmarkList landmarkList;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Add a hand landmark list to the scene to detect hand landmarks
landmarkList = gameObject.AddComponent<HandTrackingLandmarkList>();
landmarkList.enableDetection = true;
}
void Update()
{
// Get the finger landmarks from the HandTrackingLandmarkList
var thumbLandmark = landmarkList.GetLandmark(HandLandmark.ThumbTip);
var indexLandmark = landmarkList.GetLandmark(HandLandmark.IndexTip);
var middleLandmark = landmarkList.GetLandmark(HandLandmark.MiddleTip);
var ringLandmark = landmarkList.GetLandmark(HandLandmark.RingTip);
var pinkyLandmark = landmarkList.GetLandmark(HandLandmark.PinkyTip);
// Calculate the direction of each finger based on the landmarks
var thumbDirection = (thumbLandmark.WorldPosition - indexLandmark.WorldPosition).normalized;
var indexDirection = (indexLandmark.WorldPosition - middleLandmark.WorldPosition).normalized;
var middleDirection = (middleLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;
var ringDirection = (ringLandmark.WorldPosition - pinkyLandmark.WorldPosition).normalized;
var pinkyDirection = (pinkyLandmark.WorldPosition - ringLandmark.WorldPosition).normalized;
// Set the avatar's finger orientations
avatarModel.transform.Find("RightHand/Thumb").rotation = Quaternion.LookRotation(thumbDirection);
avatarModel.transform.Find("RightHand/Index").rotation = Quaternion.LookRotation(indexDirection);
avatarModel.transform.Find("RightHand/Middle").rotation = Quaternion.LookRotation(middleDirection);
avatarModel.transform.Find("RightHand/Ring").rotation = Quaternion.LookRotation(ringDirection);
avatarModel.transform.Find("RightHand/Pinky").rotation = Quaternion.LookRotation(pinkyDirection);
}
}
머리와 목
이전에 작성한 코드에서 Head와 Neck만 추가하면 머리와 목의 방향도 계산할 수 있습니다. 아래는 MediaPipe Unity Plugin을 사용하여 3D 아바타의 머리와 목의 방향을 계산하는 샘플 코드입니다.
using UnityEngine;
using Mediapipe.Unity.FaceDetection;
public class AvatarHeadNeckOrientation : MonoBehaviour
{
[SerializeField] private GameObject avatarModel;
[SerializeField] private FaceDetectionSolution faceDetection;
void Start()
{
// Load the avatar model
// You may need to adjust the position and rotation to fit the model in the scene
Instantiate(avatarModel, Vector3.zero, Quaternion.identity);
// Add a FaceDetectionSolution to the scene to detect the face
faceDetection = gameObject.AddComponent<FaceDetectionSolution>();
faceDetection.enableFaceDetection = true;
}
void Update()
{
// Get the head and neck landmarks from the FaceDetectionSolution
var headLandmark = faceDetection.faceLandmarkList?.GetLandmark(FaceLandmark.HeadTop);
var neckLandmark = faceDetection.faceLandmarkList?.GetLandmark(FaceLandmark.NoseTip);
// Calculate the direction of the head based on the landmarks
var headDirection = (headLandmark?.WorldPosition - neckLandmark?.WorldPosition)?.normalized ?? Vector3.forward;
// Calculate the direction of the neck based on the landmarks
var neckDirection = (neckLandmark?.WorldPosition - Vector3.up)?.normalized ?? Vector3.up;
// Set the avatar's head and neck orientations
avatarModel.transform.Find("Head").rotation = Quaternion.LookRotation(headDirection, neckDirection);
avatarModel.transform.Find("Neck").rotation = Quaternion.LookRotation(neckDirection);
}
}
'AI > Body Reconstruct(Mediapipe)' 카테고리의 다른 글
DepthCrafter (0) | 2024.10.04 |
---|---|
RTMPose (1) | 2024.03.04 |
One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer (0) | 2024.01.29 |
Blazepose for Avatar (3) | 2023.03.24 |
Mediapipe based Blazepose (0) | 2023.03.23 |
댓글