在我們的項目中绰沥，iOS端視頻播放使用的是第三方開源框架ijkplayer，在硬解碼的時候幔崖，解碼出的數(shù)據(jù)格式是NV12碟刺，而Unity只能渲染RGB格式，所以需要做轉(zhuǎn)換除抛。
解決方案：
在iOS端使用ffmpeg的函數(shù)將NV12格式轉(zhuǎn)換為YUV420P格式狮杨，將YUV三個分量分別傳入Unity中，然后在Unity中將YUV420P轉(zhuǎn)換為RGB并渲染到忽。
具體實現(xiàn)：
- NV12轉(zhuǎn)換為YUV420P
//獲取到NV12數(shù)據(jù)的兩個指針橄教，一個指針指向Y數(shù)據(jù)，一個指針指向UV數(shù)據(jù)
UInt8 *bufferPtrY = (UInt8 *)CVPixelBufferGetBaseAddressOfPlane(imageBuffer,0);
UInt8 *bufferPtrUV = (UInt8 *)CVPixelBufferGetBaseAddressOfPlane(imageBuffer,1);
if (!bufferPtr || !bufferPtr1) {
printf("!bufferPtrY || !bufferPtrUV\n");
return;
}
size_t width = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
size_t height = CVPixelBufferGetHeightOfPlane(imageBuffer,0);
AVFrame *pFrame=*dest_frame;
if (pFrame == NULL)
{
pFrame = av_frame_alloc();
av_image_alloc(pFrame->data, pFrame->linesize, (int)width, (int)height, AV_PIX_FMT_YUV420P,1);
*dest_frame = pFrame;
}
static int sws_flags = SWS_FAST_BILINEAR;
uint8_t * srcSlice[8] ={bufferPtrY, bufferPtrUV, NULL, NULL,NULL,NULL,NULL,NULL};
int srcStride[8] = {(int)width, (int)width, 0, 0, 0,0,0,0};
if (img_convert_ctx == NULL)
{
img_convert_ctx = sws_getContext(
(int)width,
(int)height,
AV_PIX_FMT_NV12,
(int)width,
overlay->h,
AV_PIX_FMT_YUV420P,
sws_flags,
NULL, NULL, NULL);
}
int ret = sws_scale(
img_convert_ctx,
srcSlice,
srcStride,
0,
overlay->h,
pFrame->data,
pFrame->linesize);
此時pFrame變量即包含YUV420P格式的圖像數(shù)據(jù)喘漏。
- 使用OpenGL將數(shù)據(jù)拷貝至顯存
GLuint tex[3];
// 如果tex[0]為0护蝶，那么代表所有tex還沒有初始化
if (0 == tex[0]) {
// 初始化創(chuàng)建所有紋理
for (int i = 0; i < 3; i++)
{
glGenTextures(1, &tex[i]);
glBindTexture(GL_TEXTURE_2D, tex[i]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
}
glBindTexture(GL_TEXTURE_2D, tex[0]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, w, h, 0, GL_ALPHA, GL_UNSIGNED_BYTE, pData0);
glBindTexture(GL_TEXTURE_2D, tex[1]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, w/2, h/2, 0, GL_ALPHA, GL_UNSIGNED_BYTE, pData1);
glBindTexture(GL_TEXTURE_2D, tex[2]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_ALPHA, w/2, h/2, 0, GL_ALPHA, GL_UNSIGNED_BYTE, pData2);
// 通過消息發(fā)送給Unity
UnitySendMessage(VRController::Instance()->GetRenderCallbackGoName(), VRController::Instance()->GetRenderCallbackFuncName(), [[NSString stringWithFormat:@"%ld|%ld|%ld",tex[0], tex[1], tex[2]] UTF8String]);
其中pData0、pData1翩迈、pData2分別對應YUV三個方向的分量持灰。
- Unity獲取顯存中的圖像數(shù)據(jù)
public void display(string texString) {
string[] tex = texString.Split ('|');
IntPtr nativeTex1 = (IntPtr)long.Parse (tex [0]);
IntPtr nativeTex2 = (IntPtr)long.Parse (tex [1]);
IntPtr nativeTex3 = (IntPtr)long.Parse (tex [2]);
if (_videoTextureY == null) {
_videoTextureY = Texture2D.CreateExternalTexture ((int)videoLineW, (int)videoH, TextureFormat.Alpha8,false, false, (IntPtr)nativeTex1);
_videoTextureY.filterMode = FilterMode.Bilinear;
_videoTextureY.wrapMode = TextureWrapMode.Clamp;
}
if (_videoTextureU == null) {
_videoTextureU = Texture2D.CreateExternalTexture ((int)videoLineW / 2, (int)videoH / 2, TextureFormat.Alpha8,false, false, (IntPtr)nativeTex2);
_videoTextureU.filterMode = FilterMode.Point;
_videoTextureU.wrapMode = TextureWrapMode.Clamp;
}
if (_videoTextureV == null) {
_videoTextureV = Texture2D.CreateExternalTexture ((int)videoLineW / 2, (int)videoH / 2, TextureFormat.Alpha8,false, false, (IntPtr)nativeTex3);
_videoTextureV.filterMode = FilterMode.Point;
_videoTextureV.wrapMode = TextureWrapMode.Clamp;
}
_videoTextureY.UpdateExternalTexture ((IntPtr)nativeTex1);
_videoTextureU.UpdateExternalTexture ((IntPtr)nativeTex2);
_videoTextureV.UpdateExternalTexture ((IntPtr)nativeTex3);
}
- 在shader中將YUV格式轉(zhuǎn)換為RGB格式
fixed4 yuv;
yuv.x = tex2D(_MainTexY, i.uv).a;
yuv.y = tex2D(_MainTexU, i.uv).a - 0.5;
yuv.z = tex2D(_MainTexV, i.uv).a - 0.5;
c.r = yuv.x + 1.403*yuv.z;
c.g = yuv.x - 0.344*yuv.y - 0.714*yuv.z;
c.b = yuv.x + 1.770*yuv.y;
