Hi3559AV100 NNIE开发（6）RFCN中NNIE实现关键线程函数->SAMPLE_SVP_NNIE_Rfcn_ViToVo()进行数据流分析

2021 年 3 月 25 日
笔记
hi3559, Hi35xx项目开发, RFCN线程函数分析, SAMPLE_SVP_NNIE_Rfcn_ViToVo

　　前面随笔给出了NNIE开发的基本知识，下面几篇随笔将着重于Mobilefacenet NNIE开发，实现mobilefacenet.wk的chip版本，并在Hi3559AV100上实现mobilefacenet网络功能，外接USB摄像头通过MPP平台输出至VO HDMI显示结果。下文是Hi3559AV100 NNIE开发（6）RFCN中实现关键线程函数->SAMPLE_SVP_NNIE_Rfcn_ViToVo()进行数据流分析，通过对线程函数分析，详细了解如何对.wk模型数据进行处理并弄清楚检测框绘制这些后处理的实现。

1、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函数调用

　　首先给出SAMPLE_SVP_NNIE_Rfcn_ViToVo()函数的调用，为后续分析提供参照：

 1 static pthread_t s_hNnieThread = 0; //全局定义
 2 
 3 HI_CHAR acThreadName[16] = {0};//局部定义
 4 
 5 
 6 /******************************************
 7         Create work thread
 8  ******************************************/
 9 snprintf(acThreadName, 16, "NNIE_ViToVo");
10 prctl(PR_SET_NAME, (unsigned long)acThreadName, 0,0,0);
11 pthread_create(&s_hNnieThread, 0, SAMPLE_SVP_NNIE_Rfcn_ViToVo, NULL);

　　其中prctl函数的定义如下，其中PR_SET_NAME表示使用(char *) arg2所指向的位置中的值设置调用线程的名称。

1 int prctl(int option, unsigned long arg2, unsigned long arg3,
2                  unsigned long arg4, unsigned long arg5);

2、SAMPLE_SVP_NNIE_Rfcn_ViToVo()函数实现具体分析

　　下面讲分析SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs)函数的内部实现，具体设计到5个函数，实现的功能已经注释出来，具体如下：

 1 static HI_VOID* SAMPLE_SVP_NNIE_Rfcn_ViToVo(HI_VOID* pArgs)
 2 {
 3 
 4     .... //参数定义
 5 
 6     while (HI_FALSE == s_bNnieStopSignal)
 7     {    
 8         //用户从通道获取一帧处理完成的图像 通道1-输出stExtFrmInfo
 9         s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,
10                                          as32VpssChn[1],
11                                          &stExtFrmInfo,
12                                          s32MilliSec);
13         ......
14 
15         //用户从通道获取一帧处理完成的图像 通道0-输出stBaseFrmInfo
16         s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 
17                                          as32VpssChn[0], 
18                                          &stBaseFrmInfo, 
19                                          s32MilliSec);
20         ......
21 
22         //关键处理函数
23         s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc(
24                      pstParam,
25                      pstSwParam, 
26                      &stExtFrmInfo,
27                      stBaseFrmInfo.stVFrame.u32Width,
28                      stBaseFrmInfo.stVFrame.u32Height);
29         ......
30 
31         //Draw rect
32         s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect(
33                      &stBaseFrmInfo, 
34                      &(pstSwParam->stRect), 
35                      0x0000FF00); //绿色
36     ......    
37 
38         //将视频图像送入指定输出通道显示。
39         s32Ret = HI_MPI_VO_SendFrame(voLayer, 
40                                      voChn, 
41                                      &stBaseFrmInfo, 
42                                      s32MilliSec);
43     ......
44     
45 BASE_RELEASE:
46         s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[0], &stBaseFrmInfo);
47     ......
48 
49 EXT_RELEASE:
50         s32Ret = HI_MPI_VPSS_ReleaseChnFrame(s32VpssGrp,as32VpssChn[1], &stExtFrmInfo);
51     .......
52 
53     }
54 
55     return HI_NULL;
56 }

2.1、HI_MPI_VPSS_GetChnFrame的实现

　　下面给出HI_MPI_VPSS_GetChnFrame函数的参数调用与实现过程（其中VPSS双通道输出），具体功能为从通道获取一帧处理完成的图像：

 1     HI_S32 s32VpssGrp = 0;
 2     HI_S32 as32VpssChn[] = {VPSS_CHN0, VPSS_CHN1};
 3     VIDEO_FRAME_INFO_S stBaseFrmInfo;
 4     VIDEO_FRAME_INFO_S stExtFrmInfo;
 5     HI_S32 s32MilliSec = 20000;
 6 
 7         //用户从通道获取一帧处理完成的图像 通道1-输出stExtFrmInfo
 8      s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,
 9                                 as32VpssChn[1],
10                                 &stExtFrmInfo,
11                                 s32MilliSec);
12 
13         //用户从通道获取一帧处理完成的图像 通道0-输出stBaseFrmInfo
14      s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 
15                                 as32VpssChn[0], 
16                                 &stBaseFrmInfo, 
17                                 s32MilliSec);
18 
19     //函数定义
20     HI_S32 HI_MPI_VPSS_GetChnFrame(VPSS_GRP VpssGrp, VPSS_CHN VpssChn,
21 VIDEO_FRAME_INFO_S *pstVideoFrame, HI_S32 s32MilliSec);

2.2、SAMPLE_SVP_NNIE_Rfcn_Proc的实现-NNIE数据处理

　　SAMPLE_SVP_NNIE_Rfcn_Proc函数实现是整个RFCN NNIE数据处理过程的Key Point，检测加框等信息来源处，现给出函数参数及实现分析:

 1      SAMPLE_SVP_NNIE_PARAM_S *pstParam;
 2     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 3 
 4     pstParam = &s_stRfcnNnieParam;
 5     pstSwParam = &s_stRfcnSoftwareParam;
 6 /*  
 7     s_stRfcnNnieParam及s_stRfcnSoftwareParam参数涉及前述操作
 8 
 9     s_stRfcnNnieParam.pstModel = &s_stRfcnModel.stModel;
10     s_stRfcnSoftwareParam.apcRpnDataLayerName[0] = "rpn_cls_score";
11     s_stRfcnSoftwareParam.apcRpnDataLayerName[1] = "rpn_bbox_pred";
12     s32Ret = SAMPLE_SVP_NNIE_Rfcn_ParamInit(&stNnieCfg,
13                                     &s_stRfcnNnieParam,
14                                         &s_stRfcnSoftwareParam);
15 */
16 
17 
18     VIDEO_FRAME_INFO_S stBaseFrmInfo;
19     VIDEO_FRAME_INFO_S stExtFrmInfo;
20 
21 //其中stBaseFrmInfo参数与stExtFrmInfo参数是经过下述函数输出得到
22 /*
23     s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp,
24                              as32VpssChn[1],
25                             &stExtFrmInfo,
26                             s32MilliSec);
27     s32Ret = HI_MPI_VPSS_GetChnFrame(s32VpssGrp, 
28                              as32VpssChn[0], 
29                             &stBaseFrmInfo, 
30                                 s32MilliSec);
31 */
32 
33 
34     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Proc(
35                      pstParam,
36                      pstSwParam, 
37                      &stExtFrmInfo,
38                      stBaseFrmInfo.stVFrame.u32Width,
39                      stBaseFrmInfo.stVFrame.u32Height);

　　随之给出SAMPLE_SVP_NNIE_Rfcn_Proc函数的定义及内部实现过程：

 1 static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam,
 3     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, 
 4     VIDEO_FRAME_INFO_S* pstExtFrmInfo,
 5     HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
 6 {
 7 
 8     ......参数定义
 9 
10     s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,
11                                      &stInputDataIdx,
12                                      &stProcSegIdx,
13                                      HI_TRUE);
14     
15     ......
16 
17     /*RPN*/
18     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 
19                                       pstSwParam);
20     
21 
22     if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height)
23     {
24 
25         s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
26                     pstParam,
27                     &stInputDataIdx,
28                     &pstSwParam->stRpnBbox,
29                     &stProcSegIdx,
30                     HI_TRUE);
31        ......
32         
33         s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
34                     pstParam,
35                     &stInputDataIdx,
36                     &pstSwParam->stRpnBbox,
37                     &stProcSegIdx,
38                     HI_TRUE);
39     ......
40 
41         s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam,
42                                     pstSwParam);
43         
44     }
45     else
46        { ...... }
47     s32Ret = SAMPLE_SVP_NNIE_RoiToRect(
48                 &(pstSwParam->stDstScore),
49                 &(pstSwParam->stDstRoi), 
50                 &(pstSwParam->stClassRoiNum), 
51                 pstSwParam->af32ScoreThr,
52                 HI_TRUE,
53                 &(pstSwParam->stRect),
54                 pstExtFrmInfo->stVFrame.u32Width, 
55                 pstExtFrmInfo->stVFrame.u32Height,
56                 u32BaseWidth,
57                 u32BaseHeight);
58     
59     ......
60 
61     return s32Ret;
62 
63 }

2.2.1、SAMPLE_SVP_NNIE_Forward子函数分析

　　首先给出函数的调用及参数细节，便于分析函数功能：

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中：
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam;
 3     pstParam = &s_stRfcnNnieParam;  //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc
 4     
 5 
 6 
 7     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 8         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*这个参数传进来之后经过了SP420赋值然后送入
 9                                                 SAMPLE_SVP_NNIE_Forward*/
10         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam, 
11         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
12         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
13 
14     ......
15 
16     SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S stInputDataIdx = {0};
17     SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S stProcSegIdx = {0};
18 
19     stInputDataIdx.u32SegIdx = 0;
20     stInputDataIdx.u32NodeIdx = 0;
21 
22      /*SP420*/
23     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];
24     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];
25     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride  = pstExtFrmInfo->stVFrame.u32Stride[0];
26 
27     /*NNIE process 0-th seg*/
28     stProcSegIdx.u32SegIdx = 0;
29 
30      /*NNIE process 0-th seg*/
31     stProcSegIdx.u32SegIdx = 0;
32     s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,
33                                      &stInputDataIdx,
34                                      &stProcSegIdx,
35                                      HI_TRUE);
36 
37     ......
38 
39     //函数定义
40     static HI_S32 SAMPLE_SVP_NNIE_Forward(
41                 SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam,
42                 SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx,
43                 SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,
44                 HI_BOOL bInstant)

　　此函数如其名，主要实现了NNIE forward功能，下面给出调用具体函数：

 1 /******************************************************************************
 2 * function : NNIE Forward
 3 ******************************************************************************/
 4 static HI_S32 SAMPLE_SVP_NNIE_Forward(SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam,
 5     SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx,
 6     SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,HI_BOOL bInstant)
 7 {
 8     ......
 9 
10     SAMPLE_COMM_SVP_FlushCache(
11     pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr,
12     (HI_VOID *) pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr,
13      pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size);
14 
15     /*set input blob according to node name*/
16     if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx)
17     {
18         for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++)
19         {
20             ......
21         }
22     }
23 
24     /*NNIE_Forward  多节点输入输出的 CNN 类型网络预测。 
25     对输入样本(s)进行CNN预测，对对应样本(s)进行输出响应*/
26     s32Ret = HI_MPI_SVP_NNIE_Forward(
27         &hSvpNnieHandle,
28         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc,
29         pstNnieParam->pstModel, 
30         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst,
31         &pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx], 
32         bInstant);
33     
34 
35     if(bInstant)
36     {
37         /*Wait NNIE finish,,,,enNnieId 执行网络段的 NNIE 引擎 ID。 */
38         while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].enNnieId,
39             hSvpNnieHandle, &bFinish, HI_TRUE)))
40         {
41             ......
42         }
43     }
44 
45     bFinish = HI_FALSE;
46     for(i = 0; i < pstNnieParam->astForwardCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++)
47     {
48         if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType)
49         {
50             ......
51             
52             SAMPLE_COMM_SVP_FlushCache(
53     pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
54     (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
55     u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
56 
57         }
58         else
59         {
60 
61             SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
62                 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
63                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num*
64                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn*
65                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height*
66                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
67         }
68     }
69 
70     return s32Ret;
71 }

2.2.2、SAMPLE_SVP_NNIE_Rfcn_Rpn子函数分析

　　下面给出sample_svp_NNIE_Rfcn_Rpn子函数分析，此函数主要是用于rpn相关，下面给出参数调用及函数分析：

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中：
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc
 3     pstParam = &s_stRfcnNnieParam;  //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc
 4 
 5     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 6     pstSwParam = &s_stRfcnSoftwareParam;
 7 
 8     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 9         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*这个参数传进来之后经过了SP420赋值然后送入
10                                                 SAMPLE_SVP_NNIE_Forward*/
11         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam,    //直接传给SAMPLE_SVP_NNIE_Rfcn_Rpn函数
12         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
13         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
14 
15      /*SP420*/
16     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];
17     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];
18     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride  = pstExtFrmInfo->stVFrame.u32Stride[0];
19 
20     此后，pstParam参数先传入SAMPLE_SVP_NNIE_Forward函数进行处理，随后传入SAMPLE_SVP_NNIE_Rfcn_Rpn函数
21 
22         /*RPN*/
23     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 
24                                       pstSwParam);
25 
26     // 函数调用  ，用于used to do rpn
27     HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn(
28         SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
29         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)

　　下面给出sample_svp_NNIE_Rfcn_Rpn子函数内部实现，此函数虽然传入两个参数，但是没有对pstNnieParam参数进行任何处理，完成的pstSoftwareParam结构体内大量参数的的处理，主要是调用了SVP_NNIE_Rpn函数与SAMPLE_COMM_SVP_FlushCache函数：

 1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_Rpn(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
 2     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)
 3 {
 4     HI_S32 s32Ret = HI_SUCCESS;
 5     s32Ret = SVP_NNIE_Rpn(pstSoftwareParam->aps32Conv,pstSoftwareParam->u32NumRatioAnchors,
 6         pstSoftwareParam->u32NumScaleAnchors,pstSoftwareParam->au32Scales,
 7         pstSoftwareParam->au32Ratios,pstSoftwareParam->u32OriImHeight,
 8         pstSoftwareParam->u32OriImWidth,pstSoftwareParam->au32ConvHeight,
 9         pstSoftwareParam->au32ConvWidth,pstSoftwareParam->au32ConvChannel,
10         pstSoftwareParam->u32ConvStride,pstSoftwareParam->u32MaxRoiNum,
11         pstSoftwareParam->u32MinSize,pstSoftwareParam->u32SpatialScale,
12         pstSoftwareParam->u32NmsThresh,pstSoftwareParam->u32FilterThresh,
13         pstSoftwareParam->u32NumBeforeNms,(HI_U32*)pstSoftwareParam->stRpnTmpBuf.u64VirAddr,
14         (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr,
15         &pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height);
16     SAMPLE_COMM_SVP_FlushCache(pstSoftwareParam->stRpnBbox.u64PhyAddr,
17         (HI_VOID *) pstSoftwareParam->stRpnBbox.u64VirAddr,
18         pstSoftwareParam->stRpnBbox.u32Num*
19         pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Chn*
20         pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height*
21         pstSoftwareParam->stRpnBbox.u32Stride);
22     SAMPLE_SVP_CHECK_EXPR_RET(HI_SUCCESS != s32Ret,s32Ret,SAMPLE_SVP_ERR_LEVEL_ERROR,
23         "Error,SVP_NNIE_Rpn failed!\n");
24     return s32Ret;
25 }

2.2.3、SAMPLE_SVP_NNIE_ForwardWithBbox子函数分析

　　完成前面几个函数后，通过if判断，当满足条件后执行SAMPLE_SVP_NNIE_ForwardWithBbox函数：

1 if(0 != pstSwParam->stRpnBbox.unShape.stWhc.u32Height)

　　否则不满足if条件的时候，将执行下面赋值语句：

1         for (i = 0; i < pstSwParam->stClassRoiNum.unShape.stWhc.u32Width; i++)
2         {
3             *(((HI_U32*)(HI_UL)pstSwParam->stClassRoiNum.u64VirAddr)+i) = 0;
4         }

　　而SAMPLE_SVP_NNIE_ForwardWithBbox执行了有两次，分别是对不同NNIE process x-th seg进行处理，具体如下：

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中：
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc
 3     pstParam = &s_stRfcnNnieParam;  //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc
 4 
 5     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 6     pstSwParam = &s_stRfcnSoftwareParam;
 7 
 8     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 9         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*这个参数传进来之后经过了SP420赋值然后送入
10                                                 SAMPLE_SVP_NNIE_Forward*/
11         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam,    /*直接传给SAMPLE_SVP_NNIE_Rfcn_Rpn函数
12                                                             随后传给SAMPLE_SVP_NNIE_ForwardWithBbox函数*/
13         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
14         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
15 
16     /*RPN*/
17     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam, 
18                                       pstSwParam);  /*此函数完成pstSwParam的相关赋值，并
19                                                      传给SAMPLE_SVP_NNIE_ForwardWithBbox函数*/
20 
21 
22     /*在SAMPLE_SVP_NNIE_ForwardWithBbox函数前面，已经有了下面参数的赋值，
23     并且传入至SAMPLE_SVP_NNIE_Forward函数*/
24         stInputDataIdx.u32SegIdx = 0;
25         stInputDataIdx.u32NodeIdx = 0;
26 
27      s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,
28                                      &stInputDataIdx, //stInputDataIdx参数已经用于此函数中
29                                      &stProcSegIdx,
30                                      HI_TRUE);
31 
32 
33      //函数调用
34     /*NNIE process 1-th seg, the input data comes from 3-rd report node of 0-th seg,
35      the input roi comes from RPN results*/
36     stInputDataIdx.u32SegIdx = 0;
37     stInputDataIdx.u32NodeIdx = 3;
38 
39     stProcSegIdx.u32SegIdx = 1;
40 
41     s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
42                 pstParam,
43                 &stInputDataIdx,
44                 &pstSwParam->stRpnBbox,
45                 &stProcSegIdx,
46                 HI_TRUE);
47 
48 
49     /*NNIE process 2-nd seg, the input data comes from 4-th report node of 0-th seg
50       the input roi comes from RPN results*/
51     stInputDataIdx.u32SegIdx = 0;
52     stInputDataIdx.u32NodeIdx = 4;
53 
54     stProcSegIdx.u32SegIdx = 2;
55 
56     s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
57                 pstParam,
58                 &stInputDataIdx,
59                 &pstSwParam->stRpnBbox,
60                 &stProcSegIdx,
61                 HI_TRUE);

　　下面给出函数具体实现：

 1 /******************************************************************************
 2 * function : NNIE ForwardWithBbox
 3 ******************************************************************************/
 4 static HI_S32 SAMPLE_SVP_NNIE_ForwardWithBbox(
 5             SAMPLE_SVP_NNIE_PARAM_S *pstNnieParam,
 6             SAMPLE_SVP_NNIE_INPUT_DATA_INDEX_S* pstInputDataIdx,
 7             SVP_SRC_BLOB_S astBbox[],
 8             SAMPLE_SVP_NNIE_PROCESS_SEG_INDEX_S* pstProcSegIdx,
 9             HI_BOOL bInstant)
10 {
11     ......
12 
13     SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64PhyAddr,
14         (HI_VOID *) pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u64VirAddr,
15         pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].stTskBuf.u32Size);
16 
17     /*set input blob according to node name*/
18     if(pstInputDataIdx->u32SegIdx != pstProcSegIdx->u32SegIdx)
19     {
20         for(i = 0; i < pstNnieParam->pstModel->astSeg[pstProcSegIdx->u32SegIdx].u16SrcNum; i++)
21         {
22             for(j = 0; j < pstNnieParam->pstModel->astSeg[pstInputDataIdx->u32SegIdx].u16DstNum; j++)
23             {
24                 ......
25             }
26             ......
27         }
28     }
29     /*NNIE_ForwardWithBbox*/
30     s32Ret = HI_MPI_SVP_NNIE_ForwardWithBbox(
31         &hSvpNnieHandle,
32         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astSrc,
33         astBbox,
34         pstNnieParam->pstModel,  //网络类型只支持ROI/PSROI
35         pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst,
36         &pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx], 
37         bInstant);
38 
39     ......
40 
41     if(bInstant)
42     {
43         /*Wait NNIE finish*/
44         while(HI_ERR_SVP_NNIE_QUERY_TIMEOUT == (s32Ret = HI_MPI_SVP_NNIE_Query(pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].enNnieId,
45             hSvpNnieHandle, &bFinish, HI_TRUE)))
46         {
47             ......
48         }
49     }
50 
51     bFinish = HI_FALSE;
52 
53 
54     for(i = 0; i < pstNnieParam->astForwardWithBboxCtrl[pstProcSegIdx->u32SegIdx].u32DstNum; i++)
55     {
56         if(SVP_BLOB_TYPE_SEQ_S32 == pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].enType)
57         {
58             ......
59             
60             SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
61                 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
62                 u32TotalStepNum*pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
63         }
64         else
65         {
66             SAMPLE_COMM_SVP_FlushCache(pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64PhyAddr,
67                 (HI_VOID *) pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u64VirAddr,
68                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Num*
69                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Chn*
70                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].unShape.stWhc.u32Height*
71                 pstNnieParam->astSegData[pstProcSegIdx->u32SegIdx].astDst[i].u32Stride);
72         }
73     }
74 
75     return s32Ret;
76 }

2.2.4、SAMPLE_SVP_NNIE_Rfcn_GetResult子函数分析

　　下一个子函数是获得NNIE Rfcn的结果，前提需要保证网络结构和输入数据保持一致，函数调用如下：

 1     其中在SAMPLE_SVP_NNIE_Rfcn_ViToVo函数中：
 2     SAMPLE_SVP_NNIE_PARAM_S *pstParam; //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc
 3     pstParam = &s_stRfcnNnieParam;  //此值传给SAMPLE_SVP_NNIE_Rfcn_Proc
 4 
 5     SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam;
 6     pstSwParam = &s_stRfcnSoftwareParam;
 7 
 8     static HI_S32 SAMPLE_SVP_NNIE_Rfcn_Proc(
 9         SAMPLE_SVP_NNIE_PARAM_S *pstParam,      /*这个参数传进来之后经过了SP420赋值然后送入
10                                                 SAMPLE_SVP_NNIE_Forward*/
11         SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S *pstSwParam,    /*直接传给SAMPLE_SVP_NNIE_Rfcn_Rpn函数
12                                                             随后传给SAMPLE_SVP_NNIE_ForwardWithBbox函数*/
13         VIDEO_FRAME_INFO_S* pstExtFrmInfo,
14         HI_U32 u32BaseWidth,HI_U32 u32BaseHeight)
15 
16     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64VirAddr = pstExtFrmInfo->stVFrame.u64VirAddr[0];
17     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u64PhyAddr = pstExtFrmInfo->stVFrame.u64PhyAddr[0];
18     pstParam->astSegData[stInputDataIdx.u32SegIdx].astSrc[stInputDataIdx.u32NodeIdx].u32Stride  = pstExtFrmInfo->stVFrame.u32Stride[0];
19 
20 
21     s32Ret = SAMPLE_SVP_NNIE_Forward(pstParam,   //参数调用
22                                      &stInputDataIdx,
23                                      &stProcSegIdx,
24                                      HI_TRUE);
25     
26     /*RPN*/
27     s32Ret = SAMPLE_SVP_NNIE_Rfcn_Rpn(pstParam,   //参数调用
28                                       pstSwParam);  //参数调用
29 
30 
31     s32Ret = SAMPLE_SVP_NNIE_ForwardWithBbox(
32                 pstParam,    //参数调用
33                 &stInputDataIdx,
34                 &pstSwParam->stRpnBbox,
35                 &stProcSegIdx,
36                 HI_TRUE);
37 
38 
39 
40     //函数调用
41 
42     /*GetResult*/
43     /*if user has changed net struct, please make sure SAMPLE_SVP_NNIE_Rfcn_GetResult
44      function's input datas are correct*/
45     s32Ret = SAMPLE_SVP_NNIE_Rfcn_GetResult(pstParam,
46                                             pstSwParam);
47 
48     //函数调用
49     HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
50                                       SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)

　　此函数作用是为了获取NNIE RFCN的结果，其核心是实现了SVP_NNIE_Rfcn_GetResult函数，具体如下：

 1 HI_S32 SAMPLE_SVP_NNIE_Rfcn_GetResult(SAMPLE_SVP_NNIE_PARAM_S*pstNnieParam,
 2                                       SAMPLE_SVP_NNIE_RFCN_SOFTWARE_PARAM_S* pstSoftwareParam)
 3 {
 4     HI_S32 s32Ret = HI_SUCCESS;
 5     HI_U32 i = 0;
 6     HI_S32* ps32Proposal = (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr;
 7 
 8     ......
 9 
10     for(i = 0; i < pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height; i++)
11     {
12         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i) /= SAMPLE_SVP_NNIE_QUANT_BASE;
13         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+1) /= SAMPLE_SVP_NNIE_QUANT_BASE;
14         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+2) /= SAMPLE_SVP_NNIE_QUANT_BASE;
15         *(ps32Proposal+SAMPLE_SVP_NNIE_COORDI_NUM*i+3) /= SAMPLE_SVP_NNIE_QUANT_BASE;
16     }
17     //this function is used to get RFCN result
18     s32Ret = SVP_NNIE_Rfcn_GetResult(
19                  (HI_S32*)pstNnieParam->astSegData[1].astDst[0].u64VirAddr,
20                  pstNnieParam->astSegData[1].astDst[0].u32Stride,
21                  (HI_S32*)pstNnieParam->astSegData[2].astDst[0].u64VirAddr,
22                  pstNnieParam->astSegData[2].astDst[0].u32Stride,
23                  (HI_S32*)pstSoftwareParam->stRpnBbox.u64VirAddr,
24                  pstSoftwareParam->stRpnBbox.unShape.stWhc.u32Height,
25                  pstSoftwareParam->au32ConfThresh,pstSoftwareParam->u32MaxRoiNum,
26                  pstSoftwareParam->u32ClassNum,pstSoftwareParam->u32OriImWidth,
27                  pstSoftwareParam->u32OriImHeight,pstSoftwareParam->u32ValidNmsThresh,
28                  (HI_U32*)pstSoftwareParam->stGetResultTmpBuf.u64VirAddr,
29                  (HI_S32*)pstSoftwareParam->stDstScore.u64VirAddr,
30                  (HI_S32*)pstSoftwareParam->stDstRoi.u64VirAddr,
31                  (HI_S32*)pstSoftwareParam->stClassRoiNum.u64VirAddr);
32     ......
33     return s32Ret;
34 }

2.2.5、SAMPLE_SVP_NNIE_RoiToRect子函数分析

　　SAMPLE_SVP_NNIE_Rfcn_Proc函数最后一个子函数为对ROI（感兴趣区域画框），RFNC对21类物体进行目标识别，具体调用及实现如下：

 1     /*draw result, this sample has 21 classes:
 2      class 0:background     class 1:plane           class 2:bicycle
 3      class 3:bird           class 4:boat            class 5:bottle
 4      class 6:bus            class 7:car             class 8:cat
 5      class 9:chair          class10:cow             class11:diningtable
 6      class 12:dog           class13:horse           class14:motorbike
 7      class 15:person        class16:pottedplant     class17:sheep
 8      class 18:sofa          class19:train           class20:tvmonitor*/
 9     s32Ret = SAMPLE_SVP_NNIE_RoiToRect(
10                  &(pstSwParam->stDstScore),
11                  &(pstSwParam->stDstRoi), 
12                  &(pstSwParam->stClassRoiNum), 
13                  pstSwParam->af32ScoreThr,
14                  HI_TRUE,
15                  &(pstSwParam->stRect),
16                  pstExtFrmInfo->stVFrame.u32Width, 
17                  pstExtFrmInfo->stVFrame.u32Height,
18                  u32BaseWidth,
19                  u32BaseHeight);
20 
21 /******************************************************************************
22 * function : roi to rect
23 ******************************************************************************/
24 HI_S32 SAMPLE_SVP_NNIE_RoiToRect(SVP_BLOB_S *pstDstScore,
25                                  SVP_BLOB_S *pstDstRoi, SVP_BLOB_S *pstClassRoiNum, HI_FLOAT *paf32ScoreThr,
26                                  HI_BOOL bRmBg,SAMPLE_SVP_NNIE_RECT_ARRAY_S *pstRect,
27                                  HI_U32 u32SrcWidth, HI_U32 u32SrcHeight,HI_U32 u32DstWidth,HI_U32 u32DstHeight)
28 {
29     HI_U32 i = 0, j = 0;
30     HI_U32 u32RoiNumBias = 0;
31     HI_U32 u32ScoreBias = 0;
32     HI_U32 u32BboxBias = 0;
33     HI_FLOAT f32Score = 0.0f;
34     HI_S32* ps32Score = (HI_S32*)pstDstScore->u64VirAddr;
35     HI_S32* ps32Roi = (HI_S32*)pstDstRoi->u64VirAddr;
36     HI_S32* ps32ClassRoiNum = (HI_S32*)pstClassRoiNum->u64VirAddr;
37     HI_U32 u32ClassNum = pstClassRoiNum->unShape.stWhc.u32Width;
38     HI_U32 u32RoiNumTmp = 0;
39 
40     .......
41     
42     pstRect->u32TotalNum = 0;
43     pstRect->u32ClsNum = u32ClassNum;
44     if (bRmBg)
45     {
46         pstRect->au32RoiNum[0] = 0;
47         u32RoiNumBias += ps32ClassRoiNum[0];
48         for (i = 1; i < u32ClassNum; i++)
49         {
50             u32ScoreBias = u32RoiNumBias;
51             u32BboxBias = u32RoiNumBias * SAMPLE_SVP_NNIE_COORDI_NUM;
52             u32RoiNumTmp = 0;
53             /*if the confidence score greater than result thresh, the result will be drawed*/
54             if(((HI_FLOAT)ps32Score[u32ScoreBias] / SAMPLE_SVP_NNIE_QUANT_BASE >=
55                     paf32ScoreThr[i])  &&  (ps32ClassRoiNum[i] != 0))
56             {
57                 for (j = 0; j < (HI_U32)ps32ClassRoiNum[i]; j++)
58                 {
59                     /*Score is descend order*/
60                     f32Score = (HI_FLOAT)ps32Score[u32ScoreBias + j] / SAMPLE_SVP_NNIE_QUANT_BASE;
61                     if ((f32Score < paf32ScoreThr[i]) || (u32RoiNumTmp >= SAMPLE_SVP_NNIE_MAX_ROI_NUM_OF_CLASS))
62                     {
63                         break;
64                     }
65 
66                     pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM] / (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1) ;
67                     pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 1] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1);
68 
69                     pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 2]/ (HI_FLOAT)u32SrcWidth * (HI_FLOAT)u32DstWidth) & (~1);
70                     pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32Y = pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32Y;
71 
72                     pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32X = pstRect->astRect[i][u32RoiNumTmp].astPoint[1].s32X;
73                     pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y = (HI_U32)((HI_FLOAT)ps32Roi[u32BboxBias + j*SAMPLE_SVP_NNIE_COORDI_NUM + 3] / (HI_FLOAT)u32SrcHeight * (HI_FLOAT)u32DstHeight) & (~1);
74 
75                     pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32X =  pstRect->astRect[i][u32RoiNumTmp].astPoint[0].s32X;
76                     pstRect->astRect[i][u32RoiNumTmp].astPoint[3].s32Y =  pstRect->astRect[i][u32RoiNumTmp].astPoint[2].s32Y;
77 
78                     u32RoiNumTmp++;
79                 }
80 
81             }
82 
83             pstRect->au32RoiNum[i] = u32RoiNumTmp;
84             pstRect->u32TotalNum += u32RoiNumTmp;
85             u32RoiNumBias += ps32ClassRoiNum[i];
86         }
87 
88     }
89     return HI_SUCCESS;
90 }

3、SAMPLE_COMM_SVP_NNIE_FillRect函数分析

　　SAMPLE_COMM_SVP_NNIE_FillRect主要是配合VGS实现画框功能，具体调用和实现如下，函数功能已由注释给出：

 1     //Draw rect
 2     s32Ret = SAMPLE_COMM_SVP_NNIE_FillRect(
 3                  &stBaseFrmInfo, 
 4                  &(pstSwParam->stRect), 
 5                  0x0000FF00); //绿色
 6 
 7 HI_S32 SAMPLE_COMM_SVP_NNIE_FillRect(
 8     VIDEO_FRAME_INFO_S *pstFrmInfo,
 9     SAMPLE_SVP_NNIE_RECT_ARRAY_S* pstRect, 
10     HI_U32 u32Color)
11 {
12     VGS_HANDLE VgsHandle = -1;
13     HI_S32 s32Ret = HI_SUCCESS;
14     HI_U32 i,j;
15     VGS_TASK_ATTR_S stVgsTask;//定义 VGS task 的属性
16     VGS_ADD_COVER_S stVgsAddCover;//定义 VGS 上 COVER 的配置
17     static HI_U32 u32Frm = 0;
18     u32Frm++;
19     if (0 == pstRect->u32TotalNum)
20     {
21         return s32Ret;
22     }
23     s32Ret = HI_MPI_VGS_BeginJob(&VgsHandle); //启动一个 job。
24     if (s32Ret != HI_SUCCESS)
25     {
26         ......
27         return s32Ret;
28     }
29 
30     memcpy(&stVgsTask.stImgIn, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S));
31     memcpy(&stVgsTask.stImgOut, pstFrmInfo, sizeof(VIDEO_FRAME_INFO_S));
32 
33     stVgsAddCover.enCoverType = COVER_QUAD_RANGLE;//任意四边形COVER
34     stVgsAddCover.u32Color = u32Color;  //RGB888
35     stVgsAddCover.stQuadRangle.bSolid = HI_FALSE; //空心 COVER
36     stVgsAddCover.stQuadRangle.u32Thick = 2; //2 像素对齐
37 
38     for (i = 0; i < pstRect->u32ClsNum; i++)
39     {
40         for (j = 0; j < pstRect->au32RoiNum[i]; j++)
41         {
42             memcpy(stVgsAddCover.stQuadRangle.stPoint, pstRect->astRect[i][j].astPoint, sizeof(pstRect->astRect[i][j].astPoint));
43             
44             //做 COVER 任务时，输入输出图像为同一块 buffer
45             //往一个已经启动的 job 里添加打 COVER task。 task属性必须满足VGS的能力。
46             s32Ret = HI_MPI_VGS_AddCoverTask(VgsHandle, &stVgsTask, &stVgsAddCover);
47             if (s32Ret != HI_SUCCESS)
48             {
49                 SAMPLE_PRT("HI_MPI_VGS_AddCoverTask fail,Error(%#x)\n", s32Ret);
50                 HI_MPI_VGS_CancelJob(VgsHandle);
51                 return s32Ret;
52             }
53 
54         }
55 
56     }
57     //提交一个 job。
58     s32Ret = HI_MPI_VGS_EndJob(VgsHandle);
59     if (s32Ret != HI_SUCCESS)
60     {
61         SAMPLE_PRT("HI_MPI_VGS_EndJob fail,Error(%#x)\n", s32Ret);
62         HI_MPI_VGS_CancelJob(VgsHandle);
63         return s32Ret;
64     }
65 
66     return s32Ret;

4、HI_MPI_VO_SendFrame函数分析

　　SAMPLE_SVP_NNIE_Rfcn_ViToVo线程函数执行的最后一个函数HI_MPI_VO_SendFrame，函数作用是将视频图像送入指定输出通道显示，具体调用如下：

1         s32Ret = HI_MPI_VO_SendFrame(voLayer, 
2                                      voChn, 
3                                      &stBaseFrmInfo, 
4                                      s32MilliSec);

Tags: hi3559 Hi35xx项目开发 RFCN线程函数分析 SAMPLE_SVP_NNIE_Rfcn_ViToVo