MediaPipeからBlenderへ - ② 手 - Moonlight Lullaby Note

前回の顔メッシュに続き、今度は手のデータをBlenderに取り込んでみます。

mp_to_blend.pyに手の検知のパートを追加

# mp_to_blend.py

# ========================================================================
# 準備
# ========================================================================

# bpyを外部環境でインポートする時のメッセージを防ぐ
import os, ssl
if not os.path.exists("/run/user/1000/gvfs"):
    os.mkdir("/run/user/1000/gvfs")

# Blender内部でmediapipeをインポートする時[SSL: CERTIFICATE_VERIFY_FAILED]を回避する
ssl._create_default_https_context = ssl._create_unverified_context

# ...................................................................

# 必要なモジュールを読み込む
import cv2
import mediapipe as mp
import bpy

# パス、ファイル名
PrjDir = "/Path/to/Project/"
imgName = "Input.webp"  # png, jpegも可
imgIn = PrjDir + imgName

# ...................................................................

# MediaPipeのオブジェクト
mp_face_mesh = mp.solutions.face_mesh
mp_hands = mp.solutions.hands
mp_pose = mp.solutions.pose

# 顔の各部を形成する頂点セットが予め用意されている
Face_Dict = {   "CONTOURS":mp_face_mesh.FACEMESH_CONTOURS,
                "LEFT_EYE":mp_face_mesh.FACEMESH_LEFT_EYE,
                "RIGHT_EYE":mp_face_mesh.FACEMESH_RIGHT_EYE,
                "LIPS":mp_face_mesh.FACEMESH_LIPS,
                "LEFT_EYEBROW":mp_face_mesh.FACEMESH_LEFT_EYEBROW,
                "RIGHT_EYEBROW":mp_face_mesh.FACEMESH_RIGHT_EYEBROW,
                "TESSELATION":mp_face_mesh.FACEMESH_TESSELATION,
                "FACE_OVAL":mp_face_mesh.FACEMESH_FACE_OVAL,
                "IRISES":mp_face_mesh.FACEMESH_IRISES,
                "LEFT_IRIS":mp_face_mesh.FACEMESH_LEFT_IRIS,
                "RIGHT_IRIS":mp_face_mesh.FACEMESH_RIGHT_IRIS,
                "NUM_LANDMARKS":mp_face_mesh.FACEMESH_NUM_LANDMARKS,
                "NUM_LANDMARKS_WITH_IRISES":mp_face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES
            }

hand_conn = mp_hands.HAND_CONNECTIONS

pose_conn = mp_pose.POSE_CONNECTIONS    # Blender用コネクションリストを別途作るので要らないかも

# ...................................................................

# 画像オブジェクト
img = cv2.imread(imgIn)
h = img.shape[0]
w = img.shape[1]


# ========================================================================
# サブ
# ========================================================================

def vertsCalc(h, w, lm):
    '''
    画像の高さ・幅とMediaPipeのランドマークデータから
    各頂点の座標を求める
    '''
    vList = []
    for i in lm:
        V = str(i).split("\n")
        Vx = float(V[0].split(": ")[1]) * w * 0.001
        Vy = float(V[1].split(": ")[1]) * h * 0.001
        Vz = float(V[2].split(": ")[1]) * w * 0.001
        #vList.append((Vx, Vy, Vz))
        vList.append((Vx, Vz, Vy*-1))   # Blenderではzが上、-yが前
    return vList


def edgeData(mpConn):
    '''
    MediaPipeで用意されているコネクションの頂点ペア
    '''
    pairList = []
    for i in mpConn:
        pairList.append(i)
    return pairList


# ========================================================================
# メイン
# ========================================================================

# 顔 ................................................................
with mp_face_mesh.FaceMesh(
        static_image_mode = True,
        max_num_faces = 1,
        refine_landmarks = True,
        min_detection_confidence = 0.5 ) as Face:
    results = Face.process(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
lm_detect = bool(results.multi_face_landmarks)
if lm_detect:
    Face_verts = vertsCalc(h, w, results.multi_face_landmarks[0].landmark)
    Face_edges = edgeData(Face_Dict["TESSELATION"])
    FaceMesh = bpy.data.meshes.new("face")
    FaceMesh.from_pydata(Face_verts, Face_edges, [])
    FaceMesh.validate()
    FaceMesh.update()
    ob = bpy.data.objects.new("Face", FaceMesh)
    scene = bpy.context.scene
    scene.collection.objects.link(ob)

# 手 ................................................................
with mp_hands.Hands(
        static_image_mode = True,
        max_num_hands = 2,
        min_detection_confidence = 0.5) as Hands:
    results = Hands.process(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
lm_detect = bool(results.multi_hand_landmarks)
if lm_detect:
    # 両手を認識した場合、オブジェクトを２つ作成する（手０・手１）
    for hand_No, hand_landmarks in enumerate(results.multi_hand_landmarks):
        Hand_verts = vertsCalc(h, w, results.multi_hand_landmarks[hand_No].landmark)
        Hand_edges = edgeData(hand_conn)
        HandMesh = bpy.data.meshes.new("hand"+str(hand_No))
        HandMesh.from_pydata(Hand_verts, Hand_edges, [])
        HandMesh.validate()
        HandMesh.update()
        ob = bpy.data.objects.new("Hand"+str(hand_No), HandMesh)
        scene = bpy.context.scene
        scene.collection.objects.link(ob)