手势识别示例：示例->AmebaNN->HandGestureDetection
本示例可以识别手型并实时通过rtsp进行查看，如果想进一步监测到具体的手势比如握拳（握拳是最容易识别的）需要自己编写识别代码解决。
相关库文件位于
C:\Users\用户名\AppData\Local\Arduino15\packages\realtek\hardware\AmebaPro2\4.0.9-build20250205\libraries\NeuralNetwork\src
其中NNGestureDetection.cpp，NNGestureDetection.h负责手势监测。
在NNGestureDetection.cpp中有手势组织的定义说明：
/*
# 8 12 16 20
# | | | |
# 7 11 15 19
# 4 | | | |
# | 6 10 14 18
# 3 | | | |
# | 5---9---13--17
# 2 \ /
# \ \ /
# 1 \ /
# \ \ /
# ------0-
connections = [
(0, 1), (1, 2), (2, 3), (3, 4),
(5, 6), (6, 7), (7, 8),
(9, 10), (10, 11), (11, 12),
(13, 14), (14, 15), (15, 16),
(17, 18), (18, 19), (19, 20),
(0, 5), (5, 9), (9, 13), (13, 17), (0, 17), (2,5)
*/
用21个点表示手的坐标，其中0点是手心。
计算其他点到0点的距离取平均值，当平均值小于某个阈值时，可以判断为握拳。
在NNGestureDetection.h中为NNGestureDetection增加一个成员函数
isFist(const landmark3d_t hand)用来判断是否处于握拳状态
- class NNGestureDetection: public NNModelSelection {
- public:
- static bool isFist(const landmark3d_t hand);
- }
在NNGestureDetection.cpp增加其实现函数
小于阈值90的判断为握拳
- bool NNGestureDetection::isFist(const landmark3d_t hand) {
- // 假设手掌中心是手腕点（索引0）
- const int wristIndex = 0;
- const float fistThreshold = 90; // 阈值，根据实际情况调整
- // 计算所有指尖点到手腕点的距离
- float totalDistance = 0;
- for (int i = 4; i < HAND_LANDMARK_NUM; i += 4) { // 指尖点的索引为4, 8, 12, 16, 20
- float dx = hand.pos[i].x - hand.pos[wristIndex].x;
- float dy = hand.pos[i].y - hand.pos[wristIndex].y;
- float dz = hand.pos[i].z - hand.pos[wristIndex].z;
- totalDistance += sqrt(dx * dx + dy * dy + dz * dz);
- }
- // 如果平均距离小于阈值，则认为是握拳
- float averageDistance = totalDistance / 5; // 5个指尖点
-
- char text_str[20];
- snprintf(text_str, sizeof(text_str), "averageDistance:%f",averageDistance);
- OSD.drawText(RTSP_CHANNEL, 0, 48, text_str, COLOR_BLUE);
- return averageDistance < fistThreshold;
- }
- void NNGestureDetection::drawHandObject(void *p, void *img_param)
- {
- if (!p || !img_param) {
- return;
- }
- vipnn_out_buf_t *out = (vipnn_out_buf_t *)p;
- handland_res_t *res = (handland_res_t *)&out->res[0];
- nn_data_param_t *im = (nn_data_param_t *)img_param;
- landmark3d_t lm3d=res->landmark3d;
-
- int im_h = RTSP_HEIGHT;
- int im_w = RTSP_WIDTH;
- // printf("im->width = %d, im->height = %d\r\n", im->img.width, im->img.height);
- im->img.width = im->img.height = handlandmark_nn_roi.img.width;
- float ratio_w = (float)im_w / (float)im->img.width;
- float ratio_h = (float)im_h / (float)im->img.height;
- float ratio = ratio_h < ratio_w ? ratio_h : ratio_w;
- roi_delta_qp_set_param(RTSP_CHANNEL, 0, 0, RTSP_WIDTH, RTSP_HEIGHT, ROI_DELTA_QP_MAX);
- OSD.createBitmap(RTSP_CHANNEL, HAND_JOINT_LAYER);
- OSD.createBitmap(RTSP_CHANNEL, HAND_LINK_LAYER);
- if (res->handedness != HANDEDNESS_NOTFOUND) {
- // printf("theta:%f w:%d h:%d\r\n", res->theta, res->w, res->h);
- if (fixLandmark(res) < 0) {
- return;
- }
-
-
- char text_str[20];
-
- if(res->handedness==HANDEDNESS_LEFT)
- {
- snprintf(text_str, sizeof(text_str), "LEFT");
- OSD.drawText(RTSP_CHANNEL, 0, 0, text_str, COLOR_RED);
-
- }
- else if(res->handedness==HANDEDNESS_RIGHT)
- {
- snprintf(text_str, sizeof(text_str), "RIGHT");
- OSD.drawText(RTSP_CHANNEL, 0, 0, text_str, COLOR_RED);
- }
-
- if(isFist(lm3d)){
- printf("detect fist");
- snprintf(text_str, sizeof(text_str), "FIST");
- OSD.drawText(RTSP_CHANNEL, 0, 24, text_str, COLOR_BLUE);
- }
-
- // First, draw the points for the joints.
- for (int i = 0; i < HAND_LANDMARK_NUM; i++) {
- // printf("res->landmark3d.pos[i].x = %f, res->landmark3d.pos[i].y = %f, res->landmark3d.pos[i].z = %f\r\n", res->landmark3d.pos[i].x, res->landmark3d.pos[i].y, res->landmark3d.pos[i].z);
- int xr = res->landmark3d.pos[i].x * ratio + (RTSP_WIDTH - RTSP_HEIGHT) / 2;
- int yr = res->landmark3d.pos[i].y * ratio;
- OSD.drawPoint(RTSP_CHANNEL, xr, yr, 12, COLOR_RED, HAND_JOINT_LAYER);
- // res->landmark3d.pos[i].z = roundf(q2f(&llm[(i*3+2)*datasize], llm_fmt.format));//Not needed to draw OSD
- // printf("llm %d(%f,%f,%f)\r\n", i, hand_landmark_res->landmark3d.pos[i].x, hand_landmark_res->landmark3d.pos[i].y, hand_landmark_res->landmark3d.pos[i].z);
- }
- /*
- # 8 12 16 20
- # | | | |
- # 7 11 15 19
- # 4 | | | |
- # | 6 10 14 18
- # 3 | | | |
- # | 5---9---13--17
- # 2 \ /
- # \ \ /
- # 1 \ /
- # \ \ /
- # ------0-
- connections = [
- (0, 1), (1, 2), (2, 3), (3, 4),
- (5, 6), (6, 7), (7, 8),
- (9, 10), (10, 11), (11, 12),
- (13, 14), (14, 15), (15, 16),
- (17, 18), (18, 19), (19, 20),
- (0, 5), (5, 9), (9, 13), (13, 17), (0, 17), (2,5)
- */
- int connections[22][2] = {
- {0, 1 },
- {1, 2 },
- {2, 3 },
- {3, 4 },
- {5, 6 },
- {6, 7 },
- {7, 8 },
- {9, 10},
- {10, 11},
- {11, 12},
- {13, 14},
- {14, 15},
- {15, 16},
- {17, 18},
- {18, 19},
- {19, 20},
- {0, 5 },
- {5, 9 },
- {9, 13},
- {13, 17},
- {0, 17},
- {2, 5 }
- };
- // Draw the connections of the joints
- for (int i = 0; i < 22; i++) {
- int idx_start = connections[i][0];
- int idx_end = connections[i][1];
- int start_x = res->landmark3d.pos[idx_start].x * ratio + (RTSP_WIDTH - RTSP_HEIGHT) / 2;
- int start_y = res->landmark3d.pos[idx_start].y * ratio;
- int end_x = res->landmark3d.pos[idx_end].x * ratio + (RTSP_WIDTH - RTSP_HEIGHT) / 2;
- int end_y = res->landmark3d.pos[idx_end].y * ratio;
- // printf("start_x = %d, start_y = %d, end_x = %d, end_y = %d\r\n", start_x, start_y, end_x, end_y);
- OSD.drawLine(RTSP_CHANNEL, start_x, start_y, end_x, end_y, 8, COLOR_GREEN, HAND_LINK_LAYER);
- }
- if (osd_cleanup_timer) {
- xTimerReset(osd_cleanup_timer, 10);
- }
- }
- OSD.update(RTSP_CHANNEL, HAND_JOINT_LAYER, 0);
- OSD.update(RTSP_CHANNEL, HAND_LINK_LAYER, 1);
- }
if(res->handedness==HANDEDNESS_LEFT)//判断是左手
{
snprintf(text_str, sizeof(text_str), "LEFT");
OSD.drawText(RTSP_CHANNEL, 0, 0, text_str, COLOR_RED);
}
else if(res->handedness==HANDEDNESS_RIGHT)//判断是右手
{
snprintf(text_str, sizeof(text_str), "RIGHT");
OSD.drawText(RTSP_CHANNEL, 0, 0, text_str, COLOR_RED);
}
if(isFist(lm3d)){//是否为握拳
printf("detect fist");
snprintf(text_str, sizeof(text_str), "FIST");
OSD.drawText(RTSP_CHANNEL, 0, 24, text_str, COLOR_BLUE);
}
运行效果
1、左手非握拳
屏幕显示：LEFT,平均值大于90
2、左手握拳
屏幕显示：LEFT,FIST,平均值小于90
3、右手非握拳
屏幕显示：RIGHT,平均值大于90
4、右手握拳
屏幕显示：RIGHT,FIST,平均值小于90
|
