人脸识别Mediapipe+Dlib实现1 : N式人脸匹配-526互联

一、前言

本来是想进行加速运行的。最后只快了两三帧哈哈哈哈。本次采用的方法是用Mediapipe进行人脸识别，配合使用Dlib进行特征提取以及特征向量转化。目前可以实现1 : n式人脸匹配，随便改改就行n : m了，就是效率太低了
实现原帖【---->点击这里<----】

二、难点以及之后可继续学习的领域

人脸特征向量128D转化的模型训练
人脸特征提取的训练、更加熟练使用Mediapipe
完善模型添加人员的label、无需重新训练模型的方式
使用C++重构任务

三、实现目录

进行录入数据集
数据集的向量提取、模型训练
模型使用

四、实现基本原理

使用Mediapipe进行人脸识别，识别后利用Dlib的模型，进行64个关键点的特征提取。
然后使用Dlib的模型将64个关键点的信息转化为一个128维的空间特征向量，使用多张图片得到每个人的特征向量均值。
最后计算特征向量的欧氏距离。

五、开做吧!

（1）进行截取数据集，这里使用`Mediapipe`进行获取数据集

import cv2
import os
import mediapipe as mp
import time
import sys

    # 使用mediapipe进行初始化
mp_face_detection = mp.solutions.face_detection
mp_holistic = mp.solutions.holistic
face_detection = mp_face_detection.FaceDetection(min_detection_confidence=0.5)
holistic = mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5)
def get_face(userName, imageCnt):

    # 采集的张数
    IMGCOUNT = imageCnt
    # 采集的人名
    user = userName
    # 存储位置
    output_dir = './faces/' + user #这里填编号+人名
    size = 128 # 图片边长
    # 如果文件没有就加一个
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    # 打开摄像头
    camera = cv2.VideoCapture(0)
    start_time = time.time()
    index = 1
    counter = 0
    # 获取视频宽度
    frame_width = int(camera.get(cv2.CAP_PROP_FRAME_WIDTH))
    # 获取视频高度
    frame_height = int(camera.get(cv2.CAP_PROP_FRAME_HEIGHT))
    fps = camera.get(cv2.CAP_PROP_FPS) #视频平均帧率

    while True:
        if index <= IMGCOUNT:
            # print('开始保存第 {} 张图片'.format(index))
            ref, image = camera.read()

            # 使用mediapipe进行提取特征
            # 转换BGR到RGB格式并进行人脸关键点检测
            image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            results = face_detection.process(image_rgb)
            hresults = holistic.process(image_rgb)

            # 获取roi区域
            roi = []
            if results.detections:
                for detection in results.detections:
                    bbox = detection.location_data.relative_bounding_box
                    h, w, _ = image.shape
                    x, y, w, h = int(bbox.xmin * w), int(bbox.ymin * h), \
                        int(bbox.width * w), int(bbox.height * h)

                    roi = image[y: y + h, x: x + w]
                     # 保存图片
                    face = cv2.resize(roi, (size, size))
                    cv2.imwrite(output_dir + '/' + str(index) + '.jpg', face)
                    cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
                    index += 1

            # 计算视频帧率
            counter += 1
            if (time.time() - start_time) != 0:  # 实时显示帧数

                fps = int(counter / (time.time() - start_time))
                cv2.putText(image, "FPS {:d}".format(fps), (50, 50),
                            cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255),
                            3)
                cv2.imshow('frame', image)
                counter = 0
                start_time = time.time()
            if cv2.waitKey(1) == ord('q'): break
        else:
            break

    camera.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    get_face('wxz', 100)

（2）进行将数据集提取特征点并转化为特征向量，并把特征均值放到`csv`中

import cv2
import os
import mediapipe as mp
import csv
import dlib
import numpy as np


# 返回单张图像的 128D 特征
def return_128d_features(path_img):
    img_rd = cv2.imread(path_img)
    image_rgb = cv2.cvtColor(img_rd, cv2.COLOR_BGR2RGB)
    results = face_detection.process(image_rgb)
    hresults = holistic.process(image_rgb)
    print("%-40s %-20s" % ("检测到人脸的图像 / image with faces detected:", path_img), '\n')
    # 因为有可能截下来的人脸再去检测，检测不出来人脸了
    # 所以要确保是 检测到人脸的人脸图像 拿去算特征
    if results.detections:

        bbox = results.detections[0].location_data.relative_bounding_box
        h, w, _ = img_rd.shape
        x, y, w, h = int(bbox.xmin * w), int(bbox.ymin * h), \
            int(bbox.width * w), int(bbox.height * h)
        rect = dlib.rectangle(x, y, x + w, y + h)
        shape = predictor(image_rgb, rect)
        face_descriptor = face_rec.compute_face_descriptor(image_rgb, shape)
    else:
        face_descriptor = 0
        print("no face")


    return face_descriptor


# 将文件夹中照片特征提取出来, 写入 CSV
def return_features_mean_personX(path_faces_personX, name):
    features_list_personX = []
    photos_list = os.listdir(path_faces_personX)
    if photos_list:
        for i in range(len(photos_list)):
            with open("./facedata/" + name + str(i) + ".csv", "w", # 保存每一张图片的特征的路径
                      newline="") as csvfile:
                writer = csv.writer(csvfile)
                # 调用return_128d_features()得到128d特征
                print("%-40s %-20s" % ("正在读的人脸图像 / image to read:", path_faces_personX + "/" + photos_list[i]))
                features_128d = return_128d_features(path_faces_personX + "/" + photos_list[i])
                try:
                    writer.writerow(features_128d)
                except:
                    continue
                # 遇到没有检测出人脸的图片跳过
                if features_128d != 0:
                    features_list_personX.append(features_128d)
    else:
        print("文件夹内图像文件为空 / Warning: No images in " + path_faces_personX + '/', '\n')

        # 计算 128D 特征的均值
    # N x 128D -&gt; 1 x 128D
    if features_list_personX:
        features_mean_personX = np.array(features_list_personX).mean(axis=0) # 特征均值
    else:
        features_mean_personX = '0'

    return features_mean_personX


# Dlib 人脸预测器
predictor = dlib.shape_predictor("./model/shape_predictor_68_face_landmarks.dat")

# Dlib 人脸识别模型
# Face recognition model, the object maps human faces into 128D vectors
face_rec = dlib.face_recognition_model_v1(
    "./model/dlib_face_recognition_resnet_model_v1.dat")

mp_face_detection = mp.solutions.face_detection
mp_holistic = mp.solutions.holistic
face_detection = mp_face_detection.FaceDetection(min_detection_confidence=0.5)
holistic = mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5)

# 要读取人脸图像文件的路径
path_images_from_camera = "./faces/"

people = os.listdir(path_images_from_camera)
people.sort()


with open("features_all.csv", "w", newline="") as csvfile:
    writer = csv.writer(csvfile)
    for person in people:
        print("##### " + person + " #####")
        # Get the mean/average features of face/personX, it will be a list with a length of 128D
        features_mean_personX = return_features_mean_personX(path_images_from_camera + person, person)
        writer.writerow(features_mean_personX)
        print("特征均值 / The mean of features:", list(features_mean_personX))
        print('\n')

    print("所有人脸信息保存完毕")

（3）模型的使用

幸运的是这个结果还是非常准的，因为是通过特征的向量欧氏距离进行计算的。

import os
import dlib          # 人脸处理的库 Dlib
import csv # 存入表格
import time
import numpy as np   # 数据处理的库 numpy
import cv2        # 图像处理的库 OpenCv
import pandas as pd  # 数据处理的库 Pandas
import mediapipe as mp

# 导入特征向量模型
facerec = dlib.face_recognition_model_v1("./model/dlib_face_recognition_resnet_model_v1.dat")

# 计算两个128D向量间的欧式距离
def return_euclidean_distance(feature_1, feature_2):
    feature_1 = np.array(feature_1)
    feature_2 = np.array(feature_2)
    dist = np.sqrt(np.sum(np.square(feature_1 - feature_2)))
    return dist

# 处理存放所有人脸特征的 csv
path_features_known_csv = "features_all.csv"
csv_rd = pd.read_csv(path_features_known_csv, header=None)

# 用来存放所有录入人脸特征的数组
features_known_arr = []

# 读取已知人脸数据
for i in range(csv_rd.shape[0]):
    features_someone_arr = []
    for j in range(0, len(csv_rd.loc[i, :])):
        features_someone_arr.append(csv_rd.loc[i, :][j])
    features_known_arr.append(features_someone_arr)
print("Faces in Database：", len(features_known_arr))

# 有哪些人
file_names = os.listdir("./faces")

# 使用mediapipe进行初始化
mp_face_detection = mp.solutions.face_detection
mp_holistic = mp.solutions.holistic
face_detection = mp_face_detection.FaceDetection(min_detection_confidence=0.5)
holistic = mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5)

# dlib的特征提取
predictor = dlib.shape_predictor('./model/shape_predictor_68_face_landmarks.dat')

cap = cv2.VideoCapture(0)
cap.set(3, 480)

while cap.isOpened():
    flag, img_rd = cap.read()

    # 取灰度
    image_rgb = cv2.cvtColor(img_rd, cv2.COLOR_BGR2RGB)
    results = face_detection.process(image_rgb)
    # 人脸数 faces

    # 待会要写的字体 font to write later
    font = cv2.FONT_HERSHEY_COMPLEX

    pos_namelist = []
    name_namelist = []

    if cv2.waitKey(1) == ord('q'):
        break
    # 如果识别到了人脸
    if results.detections:
        features_cap_arr = []

        bbox = results.detections[0].location_data.relative_bounding_box
        h, w, _ = img_rd.shape
        x, y, w, h = int(bbox.xmin * w), int(bbox.ymin * h), \
            int(bbox.width * w), int(bbox.height * h)

        rect = dlib.rectangle(x, y, x + w, y + h)
        shape = predictor(img_rd, rect)
        features_cap_arr.append(facerec.compute_face_descriptor(img_rd, shape))  #调用dlib的库


        pos_namelist.append(tuple([x, y - 5]))
        # 对于某张人脸，遍历所有存储的人脸特征
        e_distance_list = []

        for i in range(len(features_known_arr)):
            # 如果 person_X 数据不为空
            if str(features_known_arr[i][0]) != '0.0':
                e_distance_tmp = return_euclidean_distance(features_cap_arr[0], features_known_arr[i]) #计算两者距离
                e_distance_list.append(e_distance_tmp)
            else:
                # 空数据 person_X
                e_distance_list.append(999999999)


        # 找出最接近的一个人脸数据是第几个
        print(e_distance_list)
        similar_person_num = e_distance_list.index(min(e_distance_list)) # 距离最小的是第几个
        if min(e_distance_list) < 0.4: # 如果距离小于4
            name_namelist.append(file_names[similar_person_num])
        else:
            name_namelist.append("unknow")

        cv2.rectangle(image_rgb, (x, y), (x + w, y + h), (0, 255, 255), 2)
        cv2.putText(image_rgb, name_namelist[0], pos_namelist[0], font, 0.8, (0, 255, 255), 1, cv2.LINE_AA)
        image_rgb = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2BGR)
        cv2.imshow("camera", image_rgb)

# 释放摄像头 release camera
cap.release()
# 删除建立的窗口 delete all the windows
cv2.destroyAllWindows()