【BW21-CBV-Kit 开发板测评】手势和人脸的动态检测识别
本文介绍了安信可 BW21-CBV-Kit 开发套件实现 手势识别 和 人脸识别 的测试情况。
工程测试
介绍官方 Demo （手势监测 HandGestureDetection 和人脸识别 RTSPFaceRecegnition）测试流程及效果展示。
手势识别
通过加载 NN (Neural Network, 神经网络) 训练模型，实现手势的跟踪、提取、模拟、识别。
代码
注意修改 WiFi 名称和密码
#include "WiFi.h"
#include "StreamIO.h"
#include "VideoStream.h"
#include "RTSP.h"
#include "NNGestureDetection.h"
#include "VideoStreamOverlay.h"
#define CHANNEL 0
#define CHANNELNN 3
// Customised resolution for NN
#define NNWIDTH 192
#define NNHEIGHT 192
VideoSetting config(VIDEO_FHD, 30, VIDEO_H264, 0);
VideoSetting configNN(NNWIDTH, NNHEIGHT, 10, VIDEO_RGB, 0);
NNGestureDetection gesturedetect;
RTSP rtsp;
StreamIO videoStreamer(1, 1);
StreamIO videoStreamerRGBGD(1, 1);
char ssid[] = "xxx"; // your network SSID (name)
char pass[] = "xx"; // your network password
int status = WL_IDLE_STATUS;
IPAddress ip;
int rtsp_portnum;
void setup()
{
Serial.begin(115200);
// Attempt to connect to Wifi network:
while (status != WL_CONNECTED) {
Serial.print("Attempting to connect to WPA SSID: ");
Serial.println(ssid);
status = WiFi.begin(ssid, pass);
// wait 2 seconds for connection:
delay(2000);
}
ip = WiFi.localIP();
// Configure camera video channels with video format information
// Adjust the bitrate based on your WiFi network quality
config.setBitrate(2 * 1024 * 1024); // Recommend to use 2Mbps for RTSP streaming to prevent network congestion
Camera.configVideoChannel(CHANNEL, config);
Camera.configVideoChannel(CHANNELNN, configNN);
Camera.videoInit();
// Configure RTSP with corresponding video format information
rtsp.configVideo(config);
rtsp.begin();
rtsp_portnum = rtsp.getPort();
// Configure Gesture Detection model
// Select Neural Network(NN) task and models
gesturedetect.configVideo(configNN);
gesturedetect.modelSelect(GESTURE_DETECTION, NA_MODEL, NA_MODEL, NA_MODEL, NA_MODEL, NA_MODEL, DEFAULT_PALMDETECT, DEFAULT_HANDLANDMARK);
gesturedetect.begin();
// Configure StreamIO object to stream data from video channel to RTSP
videoStreamer.registerInput(Camera.getStream(CHANNEL));
videoStreamer.registerOutput(rtsp);
if (videoStreamer.begin() != 0) {
Serial.println("StreamIO link start failed");
}
// Start data stream from video channel
Camera.channelBegin(CHANNEL);
// Configure StreamIO object to stream data from RGB video channel to gesture detection
videoStreamerRGBGD.registerInput(Camera.getStream(CHANNELNN));
videoStreamerRGBGD.setStackSize();
videoStreamerRGBGD.setTaskPriority();
videoStreamerRGBGD.registerOutput(gesturedetect);
if (videoStreamerRGBGD.begin() != 0) {
Serial.println("StreamIO link start failed");
}
// Start video channel for NN
Camera.channelBegin(CHANNELNN);
// Start OSD drawing on RTSP video channel
OSD.configVideo(CHANNEL, config);
OSD.configTextSize(CHANNEL, 16, 32);
OSD.begin();
gesturedetect.drawHandRegion();
}
void loop()
{
// Do nothing
}
上传
1.按住 BOOT（下载） 按钮的同时按一下 EN（复位）按钮，进入下载模式；
2.选择目标串口对应的端口号，点击 Download 按钮，待上传成功提示 Success，完成固件上传；
3.再次短按 EN 键复位，执行程序。
配置
上传完成后打开串口监视器，检测到 WiFi 并连接，输出 Camera 摄像头的 ip 地址，
在 VLC 软件中输入 rtsp://192.168.1.104:554 实现 rtsp 推流。
效果
数字手势
动作手势
静态展示合集
人脸识别
使用 BW21-CBV-Kit 开发板识别检测到的人脸。
代码
详情参考：Neural Nework – Face Recognition – Realtek IoT/Wi-Fi MCU Solutions
#include "WiFi.h"
#include "StreamIO.h"
#include "VideoStream.h"
#include "RTSP.h"
#include "NNFaceDetectionRecognition.h"
#include "VideoStreamOverlay.h"
#define CHANNEL 0
#define CHANNELNN 3
// Customised resolution for NN
#define NNWIDTH 576
#define NNHEIGHT 320
VideoSetting config(VIDEO_FHD, 30, VIDEO_H264, 0);
VideoSetting configNN(NNWIDTH, NNHEIGHT, 10, VIDEO_RGB, 0);
NNFaceDetectionRecognition facerecog;
RTSP rtsp;
StreamIO videoStreamer(1, 1);
StreamIO videoStreamerFDFR(1, 1);
StreamIO videoStreamerRGBFD(1, 1);
char ssid[] = "xxx"; // your network SSID (name)
char pass[] = "xxx"; // your network password
int status = WL_IDLE_STATUS;
IPAddress ip;
int rtsp_portnum;
void setup()
{
Serial.begin(115200);
// Attempt to connect to Wifi network:
while (status != WL_CONNECTED) {
Serial.print("Attempting to connect to WPA SSID: ");
Serial.println(ssid);
status = WiFi.begin(ssid, pass);
// wait 2 seconds for connection:
delay(2000);
}
ip = WiFi.localIP();
// Configure camera video channels with video format information
// Adjust the bitrate based on your WiFi network quality
config.setBitrate(2 * 1024 * 1024); // Recommend to use 2Mbps for RTSP streaming to prevent network congestion
Camera.configVideoChannel(CHANNEL, config);
Camera.configVideoChannel(CHANNELNN, configNN);
Camera.videoInit();
// Configure RTSP with corresponding video format information
rtsp.configVideo(config);
rtsp.begin();
rtsp_portnum = rtsp.getPort();
// Configure Face Recognition model
// Select Neural Network(NN) task and models
facerecog.configVideo(configNN);
facerecog.modelSelect(FACE_RECOGNITION, NA_MODEL, DEFAULT_SCRFD, DEFAULT_MOBILEFACENET);
facerecog.begin();
facerecog.setResultCallback(FRPostProcess);
// Configure StreamIO object to stream data from video channel to RTSP
videoStreamer.registerInput(Camera.getStream(CHANNEL));
videoStreamer.registerOutput(rtsp);
if (videoStreamer.begin() != 0) {
Serial.println("StreamIO link start failed");
}
// Start data stream from video channel
Camera.channelBegin(CHANNEL);
// Configure StreamIO object to stream data from RGB video channel to face detection
videoStreamerRGBFD.registerInput(Camera.getStream(CHANNELNN));
videoStreamerRGBFD.setStackSize();
videoStreamerRGBFD.setTaskPriority();
videoStreamerRGBFD.registerOutput(facerecog);
if (videoStreamerRGBFD.begin() != 0) {
Serial.println("StreamIO link start failed");
}
// Start video channel for NN
Camera.channelBegin(CHANNELNN);
// Start OSD drawing on RTSP video channel
OSD.configVideo(CHANNEL, config);
OSD.begin();
}
void loop()
{
if (Serial.available() > 0) {
String input = Serial.readString();
input.trim();
if (input.startsWith(String("REG="))) {
String name = input.substring(4);
facerecog.registerFace(name);
} else if (input.startsWith(String("DEL="))) {
String name = input.substring(4);
facerecog.removeFace(name);
} else if (input.startsWith(String("RESET"))) {
facerecog.resetRegisteredFace();
} else if (input.startsWith(String("BACKUP"))) {
facerecog.backupRegisteredFace();
} else if (input.startsWith(String("RESTORE"))) {
facerecog.restoreRegisteredFace();
}
}
delay(2000);
OSD.createBitmap(CHANNEL);
OSD.update(CHANNEL);
}
// User callback function for post processing of face recognition results
void FRPostProcess(std::vector<FaceRecognitionResult> results)
{
uint16_t im_h = config.height();
uint16_t im_w = config.width();
Serial.print("Network URL for RTSP Streaming: ");
Serial.print("rtsp://");
Serial.print(ip);
Serial.print(":");
Serial.println(rtsp_portnum);
Serial.println(" ");
printf("Total number of faces detected = %d\r\n", facerecog.getResultCount());
OSD.createBitmap(CHANNEL);
if (facerecog.getResultCount() > 0) {
for (int i = 0; i < facerecog.getResultCount(); i++) {
FaceRecognitionResult item = results[i];
// Result coordinates are floats ranging from 0.00 to 1.00
// Multiply with RTSP resolution to get coordinates in pixels
int xmin = (int)(item.xMin() * im_w);
int xmax = (int)(item.xMax() * im_w);
int ymin = (int)(item.yMin() * im_h);
int ymax = (int)(item.yMax() * im_h);
uint32_t osd_color;
if (String(item.name()) == String("unknown")) {
osd_color = OSD_COLOR_RED;
} else {
osd_color = OSD_COLOR_GREEN;
}
// Draw boundary box
printf("Face %d name %s:\t%d %d %d %d\n\r", i, item.name(), xmin, xmax, ymin, ymax);
OSD.drawRect(CHANNEL, xmin, ymin, xmax, ymax, 3, osd_color);
// Print identification text above boundary box
char text_str[40];
snprintf(text_str, sizeof(text_str), "Face:%s", item.name());
OSD.drawText(CHANNEL, xmin, ymin - OSD.getTextHeight(CHANNEL), text_str, osd_color);
}
}
OSD.update(CHANNEL);
}
上传完成后打开串口监视器，检测到 WiFi 并连接，输出 Camera 摄像头的 ip 地址，在 VLC 软件中输入 rtsp://192.168.253.1:554 实现 rtsp 推流。
将目标人脸对准摄像头，当监测到人脸并被红色线框选中时，在串口输入 REG=Bean 实现人名录入，此时线框颜色变为绿色，提示词指示对应的人物名字。
效果
动态识别
注意到人脸识别在距离较远时准确率更高，若人脸占据画面较大比例，则识别效果下降。此外，人脸识别程序在运行过程中，模块温度有明显升高，需要注意散热等问题。
可能因素包括：代码中设置的人脸信息采集分辨率为 192x192 ，面对动态画面的实际情况，采集样本不足，对于采集画面大小、复杂的人脸表情变化的应对能力有限；摄像头分辨率限制；处理器运算能力限制等。
改进方案：增加硬件算力、散热方案、扩充样本数量、增大训练迭代次数、改进模型。
注意：若串口输出错误信息 error: vipnn not applied 则需重新安装开发板程序包。
总结
本文介绍了安信可 BW21-CBV-Kit 开发套件实现 手势识别 和 人脸识别 以及动态监测的项目，并对测试结果和试验的实际表现进行分析，给出相应的可能存在的问题和对应的解决方案，为该项目的后续完善和改进提供了思路和参考。
