????????前兩篇講了從業(yè)以來(lái)對(duì)于RTC技術(shù)悼枢，應(yīng)用場(chǎng)景以及音頻編解碼發(fā)展的理解埠忘，主要是從理論層面，所謂光說(shuō)不練假把式馒索，今天來(lái)點(diǎn)干貨莹妒，介紹一個(gè)實(shí)際應(yīng)用案例，在海思芯片上集成Aogra-RTSA SDK的開(kāi)發(fā)實(shí)踐绰上，本文設(shè)定場(chǎng)景為應(yīng)用程序控制海思芯片通過(guò)攝像頭獲取輸入視頻旨怠，經(jīng)過(guò)編碼，通過(guò)RTSA的視頻通道發(fā)送給瀏覽器觀(guān)看渔期；同時(shí)瀏覽器可以通過(guò)控制指令通知海思芯片上應(yīng)用程序运吓，抓拍圖片，然后通過(guò)數(shù)據(jù)通道發(fā)回到瀏覽器側(cè)疯趟。(實(shí)際業(yè)務(wù)使用場(chǎng)景恕本人不方便介紹)
????????首先簡(jiǎn)單介紹下海思芯片拘哨，芯片組成是ARM+編解碼協(xié)處理器，支持?jǐn)z像頭視頻接入物理接口信峻，支持圖像處理倦青，支持H.264/H.265解碼和編碼，而且在芯片里集成了視頻檢測(cè)協(xié)處理器功能模塊盹舞。
????????海思芯片的視頻及圖片處理流程如下
????????話(huà)不多說(shuō)产镐，開(kāi)始集成處理流程介紹
????????首先，搭建海思芯片的交叉編譯環(huán)境踢步，海思開(kāi)發(fā)包包含了片載運(yùn)行操作系統(tǒng)癣亚，交叉編譯開(kāi)發(fā)包，編解碼SDK包获印，已經(jīng)開(kāi)發(fā)手冊(cè)述雾，(ARM只是授權(quán)指令集和架構(gòu)，不實(shí)際生產(chǎn)兼丰，且ARM架構(gòu)系統(tǒng)很多玻孟，需要各自不同的交叉編譯環(huán)境)，此處選用在虛擬機(jī)上安裝ubuntu進(jìn)行部署(Linux ubuntu 4.15.0-128-generic #131~16.04.1-Ubuntu)鳍征，這些由于跟RTC應(yīng)用關(guān)系不大黍翎，按照開(kāi)發(fā)包中指導(dǎo)書(shū)進(jìn)行操作即可。
處理代碼主要包含5個(gè)部分
1 海思芯片硬件系統(tǒng)的初始化
2 Agora通訊通道建立
3 視頻輸入/編碼通道建立及視頻傳輸
4 抓拍通道初始化
5 抓拍指令接收及圖片傳輸
下面按照代碼對(duì)處理流程進(jìn)行一步步的講解
->首先Main函數(shù)將上述指令串接起來(lái)
HI_VOIDmain(HI_VOID)
{
???????? HI_S32 sChnCount = 1;
???????? HI_S32 s32Ret;
???????? // step:海思系統(tǒng)初始化
???????? s32Ret = Agora_Hi_SysInit();
???????? if (HI_SUCCESS != s32Ret)
???????? {
?????????????????? SAMPLE_PRT("Sys Initfailed!\n");
?????????????????? return;
???????? }
??? /******************************************
???? step : agora init and join into thechannel.
???? 初始化RTC參數(shù)并加入通訊channel
??? ******************************************/
???????? s32Ret = Agora_Chn_Init()
???????? if (HI_SUCCESS != s32Ret)
???????? {
?????????????????? SAMPLE_PRT("Agora_Chn_InitInit failed!\n");
?????????????????? return;
???????? }
??? /******************************************
???? step : stream venc init &process --create chn, get stream, then send to agora channel.
???? 創(chuàng)建視頻輸入艳丛，編碼參數(shù)匣掸，取視頻流且傳輸?shù)揭曨l通道處理
??? ******************************************/
???????? s32Ret = Agora_Hi_EncInit();
???????? if (HI_SUCCESS != s32Ret)
???????? {
?????????????????? SAMPLE_PRT("enc Initfailed!\n");
?????????????????? return;
???????? }
???????? s32Ret =Agora_Enc_StartGetStream(sChnCount);
???????? if (HI_SUCCESS != s32Ret)
???????? {
?????????????????? SAMPLE_PRT("Start Vencfailed!\n");
?????????????????? return;
???????? }
??? /******************************************
???? step : stream pic init初始化圖片抓拍通道
??? ******************************************/
???????? s32Ret = Agora_Hi_PicInit();
???????? if (HI_SUCCESS != s32Ret)
???????? {
?????????????????? SAMPLE_PRT("pic Initfailed!\n");
?????????????????? return;
???????? }
???????? while(1)
???????? {
?????????????????? // TODO: recv the config cmdfrom tcplink
?????????????????? //釋放CPU，休眠100ms氮双，處理配置不需要那么快
?????????????????? usleep(100000);
???????? }
}
->海思芯片硬件系統(tǒng)的初始化代碼如下
HI_S32Agora_Hi_SysInit(HI_VOID)
{
??? HI_S32 s32Ret = HI_SUCCESS;
??? VB_CONF_S stVbConf;
??? /******************************************
???? step?: init sys variable
??? ******************************************/
??? memset(&stVbConf,0,sizeof(VB_CONF_S));
??? stVbConf.u32MaxPoolCnt = 128;
??? /*video buffer*/
??? u32BlkSize = Agora_Sys_CalcPicVbBlkSize(gs_enNorm,\
??????????????? enSize,PIXEL_FORMAT_YUV_SEMIPLANAR_420, 16,COMPRESS_MODE_SEG);
??? stVbConf.astCommPool[0].u32BlkSize =u32BlkSize;
??? stVbConf.astCommPool[0].u32BlkCnt = 32;
??? /******************************************
???? step : mpp system init.
??? ******************************************/
??? s32Ret = MPP_Sys_Init(&stVbConf);
??? if (HI_SUCCESS != s32Ret)
??? {
??????? AGORA_TEST_PRT("system init failedwith %d!\n", s32Ret);
??????? return -1;
??? }
???????? return HI_SUCCESS;
}
->AgoraRTC通道建立代碼如下
HI_S32Agora_Chn_Init(int32_t argc, char **argv)
{
???????? int32_t rval;
??? SAMPLE_PRT("%s Welcome to RTSA SDKv%s", TAG_APP, agora_rtc_get_version());
??? // 3. API: init agora rtc sdk
??? int32_t appid_len =strlen(g_stAgoraConfig.p_appid);
??? void *p_appid = (void *)(appid_len == 0 ?NULL : g_stAgoraConfig.p_appid);
??? rval = agora_rtc_init(p_appid,g_stAgoraConfig.uid, &event_handler, g_stAgoraConfig.p_sdk_log_dir);
??? if (rval < 0) {
??????? LOGE("%s agora sdk init failed,rval=%d error=%s", TAG_API, rval, agora_rtc_err_2_str(rval));
??????? return -1;
??? }
??? // 4. API: join channel
??? int32_t token_len =strlen(g_stAgoraConfig.p_token);
??? void *p_token = (void *)(token_len == 0 ?NULL : g_stAgoraConfig.p_token);
??? rval =agora_rtc_join_channel(g_stAgoraConfig.p_channel, p_token, token_len);
??? if (rval < 0) {
??????? LOGE("%s join channel %s failed,rval=%d error=%s", TAG_API, g_stAgoraConfig.p_channel, rval, agora_rtc_err_2_str(rval));
??????? return -1;
??? }
??? // 5. wait until join channel success orCtrl-C trigger stop
??? while (1) {
??????? if (b_join_success_flag) {
??????????? break;
??????? }
??????? usleep(10000);
??? }
??? return rval;
}
->視頻輸入/編碼通道建立及視頻傳輸
????????視頻數(shù)據(jù)傳輸流程如下圖
????????編碼及傳輸最主要的有兩個(gè)步驟旺聚，1 Agora_Hi_EncInit初始化VI接收，VPSS圖像處理以及Encode編碼引擎眶蕉；2 Agora_Enc_GetVencStreamProc(線(xiàn)程處理函數(shù))砰粹，循環(huán)接收從編碼器編碼后的視頻流數(shù)據(jù)，通過(guò)Agora SDK的視頻通道傳輸?shù)絊erver造挽，供用戶(hù)端觀(guān)看碱璃。
HI_S32Agora_Hi_EncInit(HI_VOID)
{
????PAYLOAD_TYPE_E enPayLoad = PT_H264;
????PIC_SIZE_E enSize = PIC_HD1080;
????HI_U32 u32Profile = 0;
????SAMPLE_VI_MODE_E enViMode =SAMPLE_VI_MODE_8_1080P;
????HI_S32 s32VpssGrpCnt = 8;
????VPSS_GRP VpssGrp;
????VPSS_CHN VpssChn;
????VPSS_GRP_ATTR_S stVpssGrpAttr ={0};
????VENC_CHN VencChn;
????SAMPLE_RC_E enRcMode=SAMPLE_RC_CBR;
????HI_S32 s32Ret = HI_SUCCESS;
????HI_U32 u32BlkSize;
????SIZE_S stSize;
????char c;
????/******************************************
???? step : start vi dev & chn to capture
??? ******************************************/
??? s32Ret = Agora_Vi_Capture(enViMode,gs_enNorm);
??? if (HI_SUCCESS != s32Ret)
??? {
??????? AGORA_TEST_PRT("start vifailed!\n");
??????? return -1;
??? }
??? /******************************************
???? step : start vpss and vi bind vpss
??? ******************************************/
??? s32Ret = Agora_Sys_GetPicSize(gs_enNorm,enSize, &stSize);
??? if (HI_SUCCESS != s32Ret)
??? {
???????SAMPLE_PRT("SAMPLE_COMM_SYS_GetPicSize failed!\n");
??????? return -1;
??? }
??? VpssGrp = 0;
?? ? memset(&stVpssGrpAttr,0,sizeof(VPSS_GRP_ATTR_S));
???????? stVpssGrpAttr.u32MaxW?????? = stSize.u32Width;
???????? stVpssGrpAttr.u32MaxH??????? = stSize.u32Height;
???????? stVpssGrpAttr.bNrEn????? = HI_TRUE;
??? stVpssGrpAttr.enDieMode =VPSS_DIE_MODE_NODIE;
???????? stVpssGrpAttr.enPixFmt???????? = SAMPLE_PIXEL_FORMAT;
???????? s32Ret =Agora_VPSS_Start(s32VpssGrpCnt, &stSize, 1, &stVpssGrpAttr);
??? if (HI_SUCCESS != s32Ret)
??? {
???????? ???SAMPLE_PRT("Start Vpss failed!\n");
??????? return -1;
??? }
??? s32Ret = Agora_Vi_BindVpss();
??? if (HI_SUCCESS != s32Ret)
??? {
???????? ???SAMPLE_PRT("Vi bind Vpss failed!\n");
??????? return -1;
??? }
??? /******************************************
???? step : start stream venc
??? ******************************************/
??? /*** HD1080P **/
???????? VpssGrp = 0;
??? VpssChn = 0;
??? VencChn = 0;
??? s32Ret = Agora_Enc_Start(VencChn,enPayLoad,\
?????????????????????????????????? gs_enNorm,enSize, enRcMode,u32Profile);
??? if (HI_SUCCESS != s32Ret)
??? {
??????? SAMPLE_PRT("Start Vencfailed!\n");
??????? return -1;
??? }
??? s32Ret = Agora_Enc_BindVpss(VencChn,VpssGrp, VpssChn);
??? if (HI_SUCCESS != s32Ret)
??? {
??????? SAMPLE_PRT("Start Vencfailed!\n");
??????? return -1;
??? }
??? return HI_SUCCESS;
}
HI_S32Agora_Enc_StartGetStream(HI_S32 s32Cnt)
{
gs_stPara.bThreadStart = HI_TRUE;
gs_stPara.s32Cnt = s32Cnt;
return pthread_create(&gs_VencPid, 0,Agora_Enc_GetVencStreamProc, (HI_VOID*)&gs_stPara);
}
Agora_Enc_GetVencStreamProc代碼可參考GitHub上的源碼。
->抓拍通道初始化
HI_S32Agora_Hi_PicInit(HI_VOID)
{
??? HI_S32 s32Ret;
??? HI_S32 VpssGrp = 0;
??? HI_S32 VpssChn = 1;
??? HI_S32 VencChn = 1;
???????? SIZE_S stSize;
???????? // 1080P圖像
???????? stSize.u32Height = 1080;
???????? stSize.u32Width = 1920;
???????? s32Ret = Agora_Enc_SnapStart(VencChn,&stSize);
??? if (HI_SUCCESS != s32Ret)
??? {
??????? SAMPLE_PRT("Start snapfailed!\n");
??????? return -1;
??? }
???????? s32Ret = Agora_Enc_BindVpss(VencChn,VpssGrp, VpssChn);
??? if (HI_SUCCESS != s32Ret)
??? {
??????? SAMPLE_PRT("Start snap bindfailed!\n");
??????? return -1;
??? }
???????? return HI_SUCCESS;
}
->抓拍指令接收及圖片傳輸
圖片抓拍指令接收及圖片傳輸
抓拍處理分為下面幾個(gè)處理：1 注冊(cè)數(shù)據(jù)通道回調(diào)饭入，打開(kāi)數(shù)據(jù)通道嵌器；2 接收抓拍指令啟動(dòng)抓拍；3 從編碼通道接收抓拍圖片谐丢，通過(guò)數(shù)據(jù)通道傳輸給用戶(hù)存檔使用爽航。
staticvoid Agora_Hi_dchan_availability(const char *channel, int is_available)
{
??? LOGD("%s ch=%s is_available=%d",TAG_EVENT, channel, is_available);
???????? if (is_available)
???????? {
?????????????????? strncpy(datachannel, channel,31);
???????? }
}
staticvoid Agora_Hi_join_channel_success(const char *channel, int32_t elapsed)
{
??? b_join_success_flag = 1;
??? LOGI("%s join success, channel=%selapsed=%d", TAG_EVENT, channel, elapsed);
}
staticvoid Agora_Hi_on_cmd(const char *channel, uint32_t uid, int cmd, const void*param_ptr,
???????????????????????????????????? ?? size_t param_len)
{
???????? HI_S32 sRet;
??? /******************************************
???? step : get picture, then send to agoradata channel.
???? 啟動(dòng)抓拍蚓让，獲取圖片并發(fā)送到數(shù)據(jù)通道
??? ******************************************/
??? sRet = Agora_Pic_SnapProcess();
??? LOGI("Agora_Pic_SnapProcess,sRet=%d", sRet);
???????? return;
}
staticagora_rtc_event_handler_t event_handler = {
??? .on_join_channel_success???? = Agora_Hi_join_channel_success,
??? .on_rdt_availability_changed =Agora_Hi_dchan_availability,
???????? .on_cmd????????????????????????????????????????????? ?= Agora_Hi_on_cmd
};
Agora_Pic_SnapProcess可參考Github上源碼。
????????最后是代碼的實(shí)際編譯讥珍，由于海思芯片集成的是armv7架構(gòu)芯片历极，因此在選擇sdk包時(shí)，使用下面鏈接的sdk包衷佃，不然會(huì)報(bào)格式不對(duì)的錯(cuò)誤趟卸。https://download.agora.io/rtsasdk/release/AGORA-RTSALite-license-arm-linux-uclibceabi.tgz
????????Agora使用cmake編譯，按照README介紹氏义，修改toolchain.cmake中的交叉編譯器锄列，按照?qǐng)D中部分
????????更新CMakelist.txt中的.c文件以及include和so庫(kù)路徑
編譯成功，就可以運(yùn)行啦９哂啤Ａ谟省！
本文介紹了海思芯片和Agora
RTSA SDK結(jié)合完成的一個(gè)監(jiān)控視頻及圖片回傳的場(chǎng)景克婶，在后續(xù)應(yīng)用場(chǎng)景中還可以增加音頻互通以及VDA檢測(cè)后自動(dòng)抓拍回傳圖片的操作饶囚。
最后說(shuō)下整體開(kāi)發(fā)的感受：視頻/圖片編碼及獲取上，海思芯片在監(jiān)控領(lǐng)域占絕對(duì)地位鸠补，軟硬件接口豐富萝风，可以快速獲取視頻流數(shù)據(jù)，同時(shí)由于其硬件協(xié)處理器完成視頻轉(zhuǎn)碼處理紫岩，不占用ARM CPU的消耗规惰；RTC通訊上，由于A(yíng)gora的嵌入式SDK封裝的很好泉蝌，使用起來(lái)非常簡(jiǎn)單歇万，同時(shí)在瀏覽器側(cè)也有很好的支持，而且提供了多款A(yù)RM類(lèi)型的嵌入式側(cè)的SDK封裝勋陪；因此這兩個(gè)相結(jié)合可以很簡(jiǎn)單的將視頻處理和RTC傳輸應(yīng)用結(jié)合起來(lái)贪磺，搭建成一個(gè)視頻監(jiān)控傳輸系統(tǒng)(可以應(yīng)用在IPC AI防控、無(wú)人機(jī)等領(lǐng)域)诅愚。
本文GitHub源碼地址https://github.com/HansonSs82/MRTC寒锚，可參考。
本文為個(gè)人原創(chuàng)违孝，首發(fā)于聲網(wǎng)開(kāi)發(fā)者社區(qū)https://rtcdeveloper.com/t/topic/20517
