香川大学SLPからお届け！（Vol.82掲載）

著者：谷﨑勇太

　最近、さまざまな場面で「VTuber」が活躍しています。VTuberとは、2D/3Dアバターの表情や身体をリアルタイムに動かしながら動画配信をする人、あるいはそのアバターのことです。多くの場合は、カメラで人の動きを検知し、それをトレースするようにアバターを動かしています。今回は、私がPythonを使って作成した簡易VTuberシステムについて紹介します。

シェルスクリプトマガジン Vol.82は以下のリンク先でご購入できます。

図1　Webカメラの映像を読み取るためのベースコード

import cv2
camera = cv2.VideoCapture(0)
while True:
  ret,image = camera.read()
  image = cv2.cvtColor(image,BGR2RGB)
  cv2.imshow("frame", image)
  if cv2.waitKey(1) & 0xFF == ord('q'):
    break
camera.release()
cv2.destroyAllWindows()

import cv2

camera = cv2.VideoCapture(0)

while True:

ret,image = camera.read()

image = cv2.cvtColor(image,BGR2RGB)

cv2.imshow("frame", image)

if cv2.waitKey(1) & 0xFF == ord('q'):

break

camera.release()

cv2.destroyAllWindows()

図2　感情を検出するためのコード

from paz.pipelines import DetectMiniXceptionFER
pipeline = DetectMiniXceptionFER([0.1, 0.1])
output = pipeline(image)
if len(output["boxes2D"]) == 1:
  emotion = output["boxes2D"][0].class_name

from paz.pipelines import DetectMiniXceptionFER

pipeline = DetectMiniXceptionFER([0.1, 0.1])

output = pipeline(image)

if len(output["boxes2D"]) == 1:

emotion = output["boxes2D"][0].class_name

図3　左右の目の縦横比の平均値を算出する関数の定義コード

import cv2
from imutils import face_utils
import dlib
face_detector = dlib.get_frontal_face_detector()
face_parts_detector = \
  dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
def face_landmark_find(img):
  eye = 10
  img_gry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
  faces = face_detector(img_gry, 1)
  for face in faces:
    landmark = face_parts_detector(img_gry, face)
    landmark = face_utils.shape_to_np(landmark)
    left_eye_ear = calc_ear(landmark[42:48])
    right_eye_ear = calc_ear(landmark[36:42])
    eye = (left_eye_ear + right_eye_ear) / 2.0
  return eye

import cv2

from imutils import face_utils

import dlib

face_detector = dlib.get_frontal_face_detector()

face_parts_detector = \

dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

def face_landmark_find(img):

eye = 10

img_gry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

faces = face_detector(img_gry, 1)

for face in faces:

landmark = face_parts_detector(img_gry, face)

landmark = face_utils.shape_to_np(landmark)

left_eye_ear = calc_ear(landmark[42:48])

right_eye_ear = calc_ear(landmark[36:42])

eye = (left_eye_ear + right_eye_ear) / 2.0

return eye

図5　目の縦横比を計算する関数の定義コード

from scipy.spatial import distance
def calc_ear(eye):
  A = distance.euclidean(eye[1], eye[5])
  B = distance.euclidean(eye[2], eye[4])
  C = distance.euclidean(eye[0], eye[3])
  eye_ear = (A + B) / (2.0 * C)
  return eye_ear

from scipy.spatial import distance

def calc_ear(eye):

A = distance.euclidean(eye[1], eye[5])

B = distance.euclidean(eye[2], eye[4])

C = distance.euclidean(eye[0], eye[3])

eye_ear = (A + B) / (2.0 * C)

return eye_ear

図6　目の開閉を判定して処理を分岐させるコード

if w != 0:
  if eye < EYE_AR_THRESH:
    image = overlayImage(image,close,(x,y),(w,h))
  elif eye >= EYE_AR_THRESH:
    image = overlayImage(image,overlay,(x,y),(w,h))

if w != 0:

if eye < EYE_AR_THRESH:

image = overlayImage(image,close,(x,y),(w,h))

elif eye >= EYE_AR_THRESH:

image = overlayImage(image,overlay,(x,y),(w,h))

図7　顔の座標と幅の情報を取得するコード

if len(output["boxes2D"]) ==1:
  x_min, y_min, x_max, y_max = output["boxes2D"][0].coordinates
  w,h = (x_max-x_min,y_max-y_min)
  x,y =x_min,y_min

if len(output["boxes2D"]) ==1:

x_min, y_min, x_max, y_max = output["boxes2D"][0].coordinates

w,h = (x_max-x_min,y_max-y_min)

x,y =x_min,y_min

図8　overlayImage ()関数の定義コード

import cv2
from PIL import Image
import numpy as np
def overlayImage(image, overlay, location, size):
  image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
  pil_image = Image.fromarray(image)
  pil_image = pil_image.convert('RGBA')
  overlay = cv2.cvtColor(overlay, cv2.COLOR_BGRA2RGBA)
  pil_overlay = Image.fromarray(overlay)
  pil_overlay = pil_overlay.convert('RGBA')
  pil_overlay = pil_overlay.resize(size)
  pil_tmp = Image.new('RGBA', pil_image.size, (255, 255, 255, 0))
  pil_tmp.paste(pil_overlay, location, pil_overlay)
  result_image = Image.alpha_composite(pil_image, pil_tmp)
  return cv2.cvtColor(np.asarray(result_image), cv2.COLOR_RGBA2BGRA)

import cv2

from PIL import Image

import numpy as np

def overlayImage(image, overlay, location, size):

image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

pil_image = Image.fromarray(image)

pil_image = pil_image.convert('RGBA')

overlay = cv2.cvtColor(overlay, cv2.COLOR_BGRA2RGBA)

pil_overlay = Image.fromarray(overlay)

pil_overlay = pil_overlay.convert('RGBA')

pil_overlay = pil_overlay.resize(size)

pil_tmp = Image.new('RGBA', pil_image.size, (255, 255, 255, 0))

pil_tmp.paste(pil_overlay, location, pil_overlay)

result_image = Image.alpha_composite(pil_image, pil_tmp)

return cv2.cvtColor(np.asarray(result_image), cv2.COLOR_RGBA2BGRA)

図9　insert()関数の定義コード

import os
def insert():
  r = os.path.exists('user.txt')
  if r == False:
    print("初期設定を行います")
    print("目を閉じてください")
    setting()
    eye_int()
  elif r == True:
    count = len(open('user.txt').readlines())
    if count == 0:
      print("初期設定を行います")
      print("目を閉じてください")
      setting()
      eye_int()
    elif count == 1:
      eye_int()

import os

def insert():

r = os.path.exists('user.txt')

if r == False:

print("初期設定を行います")

print("目を閉じてください")

setting()

eye_int()

elif r == True:

count = len(open('user.txt').readlines())

if count == 0:

print("初期設定を行います")

print("目を閉じてください")

setting()

eye_int()

elif count == 1:

eye_int()

図10　setting ()関数の定義コード

import cv2
import math
def setting():
  f = open('user.txt', 'w')
  count = 0
  eye_sum=0
  while True:
    ret,rgb = cap.read()
    eye = face_landmark_find(rgb)
    if ret == True:
      if eye != 10:
        count +=1
        eye_sum += eye
    if(count > 50):
      x=eye_sum/50+0.01
      break
  cap.release()
  cv2.destroyAllWindows()
  x = math.floor(x*100)/100
  f.write(str(x)+'\n')
  f.close()

import cv2

import math

def setting():

f = open('user.txt', 'w')

count = 0

eye_sum=0

while True:

ret,rgb = cap.read()

eye = face_landmark_find(rgb)

if ret == True:

if eye != 10:

count +=1

eye_sum += eye

if(count > 50):

x=eye_sum/50+0.01

break

cap.release()

cv2.destroyAllWindows()

x = math.floor(x*100)/100

f.write(str(x)+'\n')

f.close()

図11　eye_int()関数の定義コード

def eye_int():
  f = open('user.txt','r+')
  lins = f.readlines()
  print('推奨する設定値は'+lins[0]+'です')
  tmp = input('設定値を入力してください:')
  f.write(tmp)

def eye_int():

f = open('user.txt','r+')

lins = f.readlines()

print('推奨する設定値は'+lins[0]+'です')

tmp = input('設定値を入力してください:')

f.write(tmp)