解碼設(shè)置
設(shè)置ItemOutput的pixelFormat為420v或者420f
- (void)p_createPlayerItemOutput {
if (!self.itemOutput) {
NSMutableDictionary *pixBuffAttributes = [NSMutableDictionary dictionary];
[pixBuffAttributes setObject:@(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
forKey:(id)kCVPixelBufferPixelFormatTypeKey];
self.itemOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:pixBuffAttributes];
[self.itemOutput setDelegate:self queue:operationQueue];
[self.itemOutput requestNotificationOfMediaDataChangeWithAdvanceInterval:ONE_FRAME_DURATION];
[_playerItem addOutput:self.itemOutput];
}
}
preview&textureCache創(chuàng)建
1旅赢、初始化MTKView
2飒硅、設(shè)置MTKView的device
- (void)p_initPreview:(CGRect)frame {
self.preivew = [[MTKView alloc] initWithFrame:frame];
self.preivew.device = MTLCreateSystemDefaultDevice();
self.preivew.delegate = self;
// self.preivew.framebufferOnly = NO;
self.viewportSize = (vector_uint2){self.preivew.drawableSize.width, self.preivew.drawableSize.height};
CVMetalTextureCacheCreate(NULL, NULL, self.preivew.device, NULL, &_textureCache);
}
pipeLine創(chuàng)建
1恢氯、通過MTKView的device創(chuàng)建Libary
2北苟、通過Libary獲取頂點函數(shù)和片源函數(shù)
3以躯、創(chuàng)建MTLRenderPipelineDescriptor,并設(shè)置頂點函數(shù)蟆盐、片源函數(shù)移必、pixelFormat
4室谚、通過MTLRenderPipelineDescriptor生成MTLRenderPipelineState，生成耗性能崔泵，須持有
5秒赤、通過MTKView的device創(chuàng)建MTLCommandQueue，生成耗性能管削，須持有
- (void)p_setupPipeline {
NSString *bundlePath = [[NSBundle bundleForClass:[self class]].resourcePath
stringByAppendingPathComponent:@"/ZZH_Sharder.bundle"];
NSBundle *resource_bundle = [NSBundle bundleWithPath:bundlePath];
NSError *error;
if (@available(iOS 10.0, *)) {
id<MTLLibrary> defaultLibrary = [self.preivew.device newDefaultLibraryWithBundle:resource_bundle error:&error];
if (error) {
ZZHDebugLog(@"%@", error);
}
id<MTLFunction> vertexFunc = [defaultLibrary newFunctionWithName:@"vertexShader"];
id<MTLFunction> fragmentFunc = [defaultLibrary newFunctionWithName:@"samplingShader"];
MTLRenderPipelineDescriptor *des = [[MTLRenderPipelineDescriptor alloc] init];
des.vertexFunction = vertexFunc;
des.fragmentFunction = fragmentFunc;
des.colorAttachments[0].pixelFormat = self.preivew.colorPixelFormat;
self.pipelineState = [self.preivew.device newRenderPipelineStateWithDescriptor:des error:nil];
self.commandQueue = [self.preivew.device newCommandQueue];
} else {
}
}
注意點
1倒脓、MTLLibary初始化
- (nullable id <MTLLibrary>)newDefaultLibrary;
- (nullable id <MTLLibrary>)newDefaultLibraryWithBundle:(NSBundle *)bundle error:(__autoreleasing NSError **)error
只會掃描mainBundle的metal文件，如果metal文件不再mainbundle需要通過指定bundle的方法初始化
初始化頂點坐標、紋理坐標
- (void)p_setupVertex {
static const ZZHVertex quadVertices[] =
{ // 頂點坐標，分別是x惶桐、y、z饲做、w； 紋理坐標遏弱，x盆均、y；
{ { 1.0, -1.0, 0.0, 1.0 }, { 1.f, 1.f } },
{ { -1.0, -1.0, 0.0, 1.0 }, { 0.f, 1.f } },
{ { -1.0, 1.0, 0.0, 1.0 }, { 0.f, 0.f } },
{ { 1.0, -1.0, 0.0, 1.0 }, { 1.f, 1.f } },
{ { -1.0, 1.0, 0.0, 1.0 }, { 0.f, 0.f } },
{ { 1.0, 1.0, 0.0, 1.0 }, { 1.f, 0.f } },
};
self.vertices = [self.preivew.device newBufferWithBytes:quadVertices
length:sizeof(quadVertices)
options:MTLResourceStorageModeShared]; // 創(chuàng)建頂點緩存
self.numVertices = sizeof(quadVertices) / sizeof(ZZHVertex); // 頂點個數(shù)
}
注意點
1漱逸、頂點坐標原點在中心點泪姨，紋理坐標原點在左下角
2、紋理的生成是由圖片像素來生成的饰抒，而圖像的存儲是從左上角開始的肮砾，對應于上圖，就是圖像左上角像素生成的紋理部分就在紋理左下角處袋坑，即 上下顛倒仗处。
設(shè)置轉(zhuǎn)換矩陣
- (void)p_setupMatrix { // 設(shè)置好轉(zhuǎn)換的矩陣
matrix_float3x3 kColorConversion601FullRangeMatrix = (matrix_float3x3){
(simd_float3){1.164, 1.164, 1.164},
(simd_float3){0.0, -0.213, 2.112},
(simd_float3){1.793, -0.533, 0.0},
};
// shader 會將數(shù)據(jù)歸一化，而 uv 的取值區(qū)間本身存在-128到正128 然后歸一化到0-1 為了正確計算成rgb，
// 則需要歸一化到 -0.5 - 0.5的區(qū)間
vector_float3 kColorConversion601VideoRangeOffset = (vector_float3){ 0, -0.5, -0.5}; // 這個是偏移
ZZHConvertMatrix matrix;
// 設(shè)置參數(shù)
matrix.matrix = kColorConversion601FullRangeMatrix;
matrix.offset = kColorConversion601VideoRangeOffset;
self.convertMatrix = [self.preivew.device newBufferWithBytes:&matrix
length:sizeof(ZZHConvertMatrix)
options:MTLResourceStorageModeShared];
}
注意點
1婆誓、YUV和rgb的轉(zhuǎn)換矩陣主要依賴yuv的顏色空間決策
2吃环、shader 會將數(shù)據(jù)歸一化，而 uv 的取值區(qū)間本身存在-128到正128 然后歸一化到0-1 為了正確計算成rgb洋幻， 則需要歸一化到 -0.5 - 0.5的區(qū)間郁轻。同時對VideoRange的y需要減掉16/255.0
幀渲染
1、通過commandQuene生成當前這次渲染所需要的MTLCommandBuffer
2鞋屈、通過MTKView獲取MTLRenderPassDescriptor
3范咨、設(shè)置MTLRenderPassDescriptor的clearColor
4故觅、通過MTLCommandBuffer和MTLRenderPassDescriptor生成MTLRenderCommandEncoder
5厂庇、encoder設(shè)置viewPort
6、encoder設(shè)置renderPipelineState
7输吏、encoder設(shè)置頂點buffer
8权旷、encoder上傳紋理
9、encoder設(shè)置片源buffer(轉(zhuǎn)換矩陣)
10贯溅、encoder drawing
11拄氯、encoder endEncoding
12、commandBuffer 顯示
13它浅、commandBuffer commit
- (void)displayPixelBuffer:(CVPixelBufferRef)pixelBuffer {
/* 兩種方法獲取顏色空間信息
CFTypeRef colorAttachments = CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL);
OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
CFDictionaryRef cfMetadataDic = CMCopyDictionaryOfAttachments(NULL, pixelBuffer, kCMAttachmentMode_ShouldPropagate);
NSDictionary *nsMetadataDic = (__bridge NSDictionary *_Nonnull)(cfMetadataDic);
//發(fā)現(xiàn)用pixelFormatType創(chuàng)建的format跟pixelbuffer的format是一樣的
CFDictionaryRef formatDes = CVPixelFormatDescriptionCreateWithPixelFormatType(kCFAllocatorDefault, CVPixelBufferGetPixelFormatType(pixelBuffer));
**/
id<MTLCommandBuffer> commandBuffer = [self.commandQueue commandBuffer];
MTLRenderPassDescriptor *renderPassDes = self.preivew.currentRenderPassDescriptor;
if (renderPassDes && pixelBuffer) {
renderPassDes.colorAttachments[0].clearColor = MTLClearColorMake(0, 0.f, 0.f, 1.f);
id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:renderPassDes];
[renderEncoder setViewport:(MTLViewport){0.0, 0.0, self.viewportSize.x, self.viewportSize.y, -1.0, 1.0 }]; // 設(shè)置顯示區(qū)域
[renderEncoder setRenderPipelineState:self.pipelineState]; // 設(shè)置渲染管道译柏，以保證頂點和片元兩個shader會被調(diào)用
[renderEncoder setVertexBuffer:self.vertices offset:0 atIndex:ZZHVertexInputIndexVertices];
[self p_setupTextureWithEncoder:renderEncoder pixelBuffer:pixelBuffer];
[renderEncoder setFragmentBuffer:self.convertMatrix offset:0 atIndex:ZZHFragmentInputIndexMatrix];
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:self.numVertices];
[renderEncoder endEncoding];
[commandBuffer presentDrawable:self.preivew.currentDrawable];
}
[commandBuffer commit];
}
紋理上傳
- (void)p_setupTextureWithEncoder:(id<MTLRenderCommandEncoder>)encoder pixelBuffer:(CVPixelBufferRef)pixelBuffer {
id<MTLTexture> textureY = nil;
id<MTLTexture> textureUV = nil;
size_t width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
size_t height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
CVMetalTextureRef yTexture = NULL;
CVReturn status = CVMetalTextureCacheCreateTextureFromImage(NULL, self.textureCache, pixelBuffer, NULL, MTLPixelFormatR8Unorm, width, height, 0, &yTexture);
if (status == kCVReturnSuccess) {
textureY = CVMetalTextureGetTexture(yTexture);
CFRelease(yTexture);
}
width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);
height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);
CVMetalTextureRef uvTexture = NULL;
status = CVMetalTextureCacheCreateTextureFromImage(NULL, self.textureCache, pixelBuffer, NULL, MTLPixelFormatRG8Unorm, width, height, 1, &uvTexture);
if (status == kCVReturnSuccess) {
textureUV = CVMetalTextureGetTexture(uvTexture);
CFRelease(uvTexture);
}
if (textureY && textureUV) {
[encoder setFragmentTexture:textureY atIndex:ZZHFragmentTextureIndexTextureY];
[encoder setFragmentTexture:textureUV atIndex:ZZHFragmentTextureIndexTextureUV];
}
}
注意點
1、uv數(shù)據(jù)是通過RG格式上傳到一個紋理的
2姐霍、紋理寬高需要是當前plane的寬高
metal
typedef struct {
float4 clipSpacePosition [[position]];
float2 textureCoordinate;
} RasterizerData;
// vertex_id是頂點shader每次處理的index鄙麦，用于定位當前的頂點
// buffer表明是緩存數(shù)據(jù)，ZZHVertexInputIndexVertices是索引
vertex RasterizerData vertexShader(uint vertexID [[vertex_id]], constant ZZHVertex *vertexArray [[buffer(ZZHVertexInputIndexVertices)]]) {
RasterizerData out;
out.clipSpacePosition = vertexArray[vertexID].position;
out.textureCoordinate = vertexArray[vertexID].textureCoordinate;
return out;
}
//stage_in :標識該數(shù)據(jù)來自vertex的輸出
// texture表明是紋理數(shù)據(jù)镊折，LYFragmentTextureIndexTextureY是索引
fragment float4 samplingShader(RasterizerData input [[stage_in]], texture2d<float> textureY [[texture(ZZHFragmentTextureIndexTextureY)]], texture2d<float> textureUV [[texture(ZZHFragmentTextureIndexTextureUV)]], constant ZZHConvertMatrix *convertMatrix [[buffer(ZZHFragmentInputIndexMatrix)]]) {
constexpr sampler textureSampler (mag_filter::linear, min_filter::linear);
float3 yuv = float3(textureY.sample(textureSampler, input.textureCoordinate).r, textureUV.sample(textureSampler, input.textureCoordinate).rg);
float3 rgb = convertMatrix->matrix * (yuv + convertMatrix->offset);
return float4(rgb, 1.0);
}
注意點
1胯府、metal引用pod文件需要通過modlemap的方式引用
2、[[buffer(ZZHFragmentInputIndexMatrix)]] : buffer表明是緩存數(shù)據(jù)恨胚，ZZHVertexInputIndexVertices是索引骂因。對應encoder的接口
- (void)setFragmentBuffer:(nullable id <MTLBuffer>)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index;
- (void)setVertexBuffer:(nullable id <MTLBuffer>)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index;
3、[[vertex_id]] ：vertex_id是頂點shader每次處理的index赃泡，用于定位當前的頂點
4寒波、[[stage_in]] ： stage_in :標識該數(shù)據(jù)來自vertex的輸出
5、Metal中的內(nèi)存訪問主要有兩種方式：Device模式和Constant模式升熊，由代碼中顯式指定俄烁。
Device模式是比較通用的訪問模式，使用限制比較少僚碎，而Constant模式是為了多次讀取而設(shè)計的快速訪問只讀模式猴娩，通過Constant內(nèi)存模式訪問的參數(shù)的數(shù)據(jù)的字節(jié)數(shù)量是固定的，特點總結(jié)為：
Device支持讀寫，并且沒有size的限制卷中；
Constant是只讀矛双，并且限定大小蟆豫；
如何選擇Device和Constant模式议忽？
先看數(shù)據(jù)size是否會變化，再看訪問的頻率高低十减，只有那些固定size且經(jīng)常訪問的部分適合使用constant模式栈幸，其他的均用Device。
6帮辟、Sampler是采樣器速址，決定如何對一個紋理進行采樣操作。尋址模式由驹，過濾模式芍锚，歸一化坐標，比較函數(shù)蔓榄。
在Metal程序里初始化的采樣器必須使用constexpr修飾符聲明并炮。
采樣器指針和引用是不支持的，將會導致編譯錯誤甥郑。
