本文檔將介紹iOS Camera回調(diào)的視頻數(shù)據(jù)如何轉(zhuǎn)換為后續(xù)OpenGL圖像渲染所需要的texture溉愁，并介紹幾種常用顏色存儲(chǔ)的數(shù)據(jù)形式轉(zhuǎn)換texture的方式仆救。
1秸抚、Camera回調(diào)視頻數(shù)據(jù)
camera的回調(diào)數(shù)據(jù)接口：
- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection;
回調(diào)回來sampleBuffer數(shù)據(jù)：
CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)CMSampleBufferGetImageBuffer(sampleBuffer);
OSType result = CVPixelBufferGetPixelFormatType(pixelBuffer);
通過CVPixelBufferGetPixelFormatType看到所獲取視頻數(shù)據(jù)類型感帅，數(shù)據(jù)類型其實(shí)是在配置Camera時(shí)候設(shè)置的：
@property(nonatomic, copy) NSDictionary *videoSettings;
常用的3中類型為：
kCVPixelFormatType_32RGBA = 'RGBA', /* 32 bit RGBA */
kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange = '420v', /* Bi-Planar Component Y'CbCr 8-bit 4:2:0, video-range (luma=[16,235] chroma=[16,240]). baseAddr points to a big-endian CVPlanarPixelBufferInfo_YCbCrBiPlanar struct */
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange = '420f', /* Bi-Planar Component Y'CbCr 8-bit 4:2:0, full-range (luma=[0,255] chroma=[1,255]). baseAddr points to a big-endian CVPlanarPixelBufferInfo_YCbCrBiPlanar struct */
后兩種其實(shí)就是視頻處理中的420sp圆雁，在ios中排列形式都是NV12，接下來我會(huì)對(duì)以上的三種形式和yuv420p進(jìn)行texture的生成为黎。
2邮丰、 BGRA8888形式的綁定
- (BOOL)setupOriginTextureWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
{
CVReturn cvRet = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_cvTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RGBA,
self.imageWidth,
self.imageHeight,
GL_BGRA,
GL_UNSIGNED_BYTE,
0,
&_cvTextureOrigin);
if (!_cvTextureOrigin || kCVReturnSuccess != cvRet) {
NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage %d" , cvRet);
return NO;
}
_textureOriginInput = CVOpenGLESTextureGetName(_cvTextureOrigin);
glBindTexture(GL_TEXTURE_2D , _textureOriginInput);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glBindTexture(GL_TEXTURE_2D, 0);
return YES;
}
通過使用CVOpenGLESTextureCacheCreateTextureFromImage，將BGRA的數(shù)據(jù)轉(zhuǎn)換到RGBA的紋理上铭乾。
3剪廉、對(duì)于420f或者420v形式的綁定
使用CVOpenGLESTextureCacheCreateTextureFromImage創(chuàng)建Y、UV兩個(gè)紋理炕檩，并在fragment shader中轉(zhuǎn)換為RGBA
- (BOOL)setupLumaTextureWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
{
//Y-plane
glActiveTexture(GL_TEXTURE0);
CVReturn cvRet = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_cvTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_LUMINANCE,
self.imageWidth,
self.imageHeight,
GL_LUMINANCE,
GL_UNSIGNED_BYTE,
0,
&_cvlumaTexture);
if (!_cvlumaTexture || kCVReturnSuccess != cvRet) {
NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage %d" , cvRet);
return NO;
}
_textureLuma = CVOpenGLESTextureGetName(_cvlumaTexture);
glBindTexture(CVOpenGLESTextureGetTarget(_cvlumaTexture), _textureLuma);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//glBindTexture(GL_TEXTURE_2D, 0);
// UV-plane.
glActiveTexture(GL_TEXTURE0);
cvRet = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_cvTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_LUMINANCE_ALPHA,
self.imageWidth / 2,
self.imageHeight / 2,
GL_LUMINANCE_ALPHA,
GL_UNSIGNED_BYTE,
1,
&_cvchromaTexture);
if (!_cvchromaTexture || kCVReturnSuccess != cvRet) {
NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage %d" , cvRet);
return NO;
}
_textureChroma = CVOpenGLESTextureGetName(_cvchromaTexture);
glBindTexture(CVOpenGLESTextureGetTarget(_cvchromaTexture), _textureChroma);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//glBindTexture(GL_TEXTURE_2D, 0);
return YES;
}
internalFormat斗蒋、format參數(shù)設(shè)置：GL_LUMINANCE 、GL_LUMINANCE_ALPHA
可以嘗試GL_RED_EXT笛质、GL_RG_EXT泉沾，應(yīng)該會(huì)有顏色上的變化。
對(duì)應(yīng)的fragment shader
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange：
//yuv420f/v
char fsh1[] = "varying highp vec2 textureCoordinate;\
precision mediump float;\
uniform sampler2D SamplerY;\
uniform sampler2D SamplerUV;\
uniform mat3 colorConversionMatrix;\
void main()\
{\
mediump vec3 yuv;\
lowp vec3 rgb;\
yuv.x = (texture2D(SamplerY, textureCoordinate).r);\
yuv.yz = (texture2D(SamplerUV, textureCoordinate).ra - vec2(0.5, 0.5));\
rgb = colorConversionMatrix * yuv;\
gl_FragColor = vec4(rgb,1);\
}";
kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange：
//yuv420f/v
char fsh4[] = "varying highp vec2 textureCoordinate;\
precision mediump float;\
uniform sampler2D SamplerY;\
uniform sampler2D SamplerUV;\
uniform mat3 colorConversionMatrix;\
void main()\
{\
mediump vec3 yuv;\
lowp vec3 rgb;\
yuv.x = (texture2D(SamplerY, textureCoordinate).r - (16.0 / 255.0));\
yuv.yz = (texture2D(SamplerUV, textureCoordinate).ra - vec2(0.5, 0.5));\
rgb = colorConversionMatrix * yuv;\
gl_FragColor = vec4(rgb,1);\
}";
4妇押、yuv420p
//glGenTextures(1, &(_textureY));
glBindTexture(GL_TEXTURE_2D, _textureY);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, self.imageWidth, self.imageHeight, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, yuv_Y);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//glGenTextures(1, &(_textureU));
glBindTexture(GL_TEXTURE_2D, _textureU);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, self.imageWidth/2, self.imageHeight/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, yuv_U);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//glGenTextures(1, &(_textureV));
glBindTexture(GL_TEXTURE_2D, _textureV);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, self.imageWidth/2, self.imageHeight/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, yuv_V);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
對(duì)應(yīng)的fragment shader：
char fsh3[] = "varying highp vec2 textureCoordinate;\
precision mediump float;\
uniform sampler2D SamplerY;\
uniform sampler2D SamplerU;\
uniform sampler2D SamplerV;\
uniform mat3 colorConversionMatrix;\
void main()\
{\
mediump vec3 yuv;\
lowp vec3 rgb;\
yuv.x = (texture2D(SamplerY, textureCoordinate).r - (16.0 / 255.0));\
yuv.y = (texture2D(SamplerU, textureCoordinate).r - 0.5);\
yuv.z = (texture2D(SamplerV, textureCoordinate).r - 0.5);\
rgb = colorConversionMatrix * yuv;\
gl_FragColor = vec4(rgb,1);\
}";
5跷究、調(diào)試
首先可以通過Debug/Capture GPU Frame來抓取一幀畫面
Snip20170809_7.png
這樣可以看到Y(jié)、UV紋理有沒有創(chuàng)建成功敲霍，并且給出類RenderBuffer的渲染情況俊马。
每一幀數(shù)據(jù)都需要flush
if (_cvchromaTexture) {
CFRelease(_cvchromaTexture);
_cvchromaTexture = NULL;
}
if (_cvlumaTexture) {
CFRelease(_cvlumaTexture);
_cvlumaTexture = NULL;
}