ios 利用 socket 傳輸 replykit 屏幕共享數(shù)據(jù)到主 app
先上 demo
我這里只講代碼，文章知識(shí)點(diǎn)什么的逛绵，大家自己搜索，網(wǎng)上太多了乌企，比我說(shuō)的好
1. replykit 使用
//
// ViewController.m
// Socket_Replykit
//
// Created by 孫承秀 on 2020/5/19.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "ViewController.h"
#import <ReplayKit/ReplayKit.h>
#import "RongRTCServerSocket.h"
@interface ViewController ()<RongRTCServerSocketProtocol>
@property (nonatomic, strong) RPSystemBroadcastPickerView *systemBroadcastPickerView;
/**
server socket
*/
@property(nonatomic , strong)RongRTCServerSocket *serverSocket;
@end
@implementation ViewController
- (void)viewDidLoad {
[super viewDidLoad];
self.view.backgroundColor = [UIColor whiteColor];
// Do any additional setup after loading the view.
[self.serverSocket createServerSocket];
self.systemBroadcastPickerView = [[RPSystemBroadcastPickerView alloc] initWithFrame:CGRectMake(0, 64, [UIScreen mainScreen].bounds.size.width, 80)];
self.systemBroadcastPickerView.preferredExtension = @"cn.rongcloud.sealrtc.RongRTCRP";
self.systemBroadcastPickerView.backgroundColor = [UIColor colorWithRed:53.0/255.0 green:129.0/255.0 blue:242.0/255.0 alpha:1.0];
self.systemBroadcastPickerView.showsMicrophoneButton = NO;
[self.view addSubview:self.systemBroadcastPickerView];
}
-(RongRTCServerSocket *)serverSocket{
if (!_serverSocket) {
RongRTCServerSocket *socket = [[RongRTCServerSocket alloc] init];
socket.delegate = self;
_serverSocket = socket;
}
return _serverSocket;
}
-(void)didProcessSampleBuffer:(CMSampleBufferRef)sampleBuffer{
// 這里拿到了最終的數(shù)據(jù)阁最，比如最后可以使用融云的音視頻SDK RTCLib 進(jìn)行傳輸就可以了
}
@end
@end
打開(kāi)一個(gè)屏幕共享就是這么容易,
其中戒祠，也包括了，創(chuàng)建 server soket 的步驟速种，我們把主app當(dāng)做server姜盈，然后屏幕共享 extension 當(dāng)做 client ，通過(guò)socket像我們主app發(fā)送數(shù)據(jù)
在extension 里面配阵，我們拿到屏幕共享數(shù)據(jù)之后
//
// SampleHandler.m
// SocketReply
//
// Created by 孫承秀 on 2020/5/19.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "SampleHandler.h"
#import "RongRTCClientSocket.h"
@interface SampleHandler()
/**
client servert
*/
@property(nonatomic , strong)RongRTCClientSocket *clientSocket;
@end
@implementation SampleHandler
- (void)broadcastStartedWithSetupInfo:(NSDictionary<NSString *,NSObject *> *)setupInfo {
// User has requested to start the broadcast. Setup info from the UI extension can be supplied but optional.
self.clientSocket = [[RongRTCClientSocket alloc] init];
[self.clientSocket createCliectSocket];
}
- (void)broadcastPaused {
// User has requested to pause the broadcast. Samples will stop being delivered.
}
- (void)broadcastResumed {
// User has requested to resume the broadcast. Samples delivery will resume.
}
- (void)broadcastFinished {
// User has requested to finish the broadcast.
}
- (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer withType:(RPSampleBufferType)sampleBufferType {
switch (sampleBufferType) {
case RPSampleBufferTypeVideo:
// Handle video sample buffer
[self sendData:sampleBuffer];
break;
case RPSampleBufferTypeAudioApp:
// Handle audio sample buffer for app audio
break;
case RPSampleBufferTypeAudioMic:
// Handle audio sample buffer for mic audio
break;
default:
break;
}
}
- (void)sendData:(CMSampleBufferRef)sampleBuffer{
[self.clientSocket encodeBuffer:sampleBuffer];
}
@end
可見(jiàn) 馏颂，這里我們創(chuàng)建了一個(gè) client socket，然后拿到屏幕共享的視頻buffer之后棋傍，通過(guò)socket發(fā)給我們的主app饱亮，這就是屏幕共享額流程
2. local socket 的使用
//
// RongRTCSocket.m
// SealRTC
//
// Created by 孫承秀 on 2020/5/7.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "RongRTCSocket.h"
#import <arpa/inet.h>
#import <netdb.h>
#import <sys/types.h>
#import <sys/socket.h>
#import <ifaddrs.h>
#import "RongRTCThread.h"
@interface RongRTCSocket()
/**
rec thread
*/
@property(nonatomic , strong)RongRTCThread *recvThread;
@end
@implementation RongRTCSocket
- (int)createSocket{
int sock = socket(AF_INET, SOCK_STREAM, 0);
self.sock = sock;
if (self.sock == -1) {
close(self.sock);
NSLog(@"??????????socket error : %d",self.sock);
}
self.recvThread = [[RongRTCThread alloc] init];
[self.recvThread run];
return sock;
}
- (void)setSendBuffer{
int optVal = 1024 * 1024 * 2;
int optLen = sizeof(int);
int res = setsockopt(self.sock, SOL_SOCKET,SO_SNDBUF,(char*)&optVal,optLen );
NSLog(@"??????????set send buffer:%d",res);
}
- (void)setRecvBuffer{
int optVal = 1024 * 1024 * 2;
int optLen = sizeof(int);
int res = setsockopt(self.sock, SOL_SOCKET,SO_RCVBUF,(char*)&optVal,optLen );;
NSLog(@"??????????set send buffer:%d",res);
}
- (void)setSendingTimeout{
struct timeval timeout = {10,0};
int res = setsockopt(self.sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(int));
NSLog(@"??????????set send timeout:%d",res);
}
- (void)setRecvTimeout{
struct timeval timeout = {10,0};
int res = setsockopt(self.sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(int));
NSLog(@"??????????set send timeout:%d",res);
}
- (BOOL)connect{
NSString *serverHost = [self ip];
struct hostent *server = gethostbyname([serverHost UTF8String]);
if (server == NULL) {
close(self.sock);
NSLog(@"??????????get host error");
return NO;
}
struct in_addr *remoteAddr = (struct in_addr *)server->h_addr_list[0];
struct sockaddr_in addr;
addr.sin_family = AF_INET;
addr.sin_addr = *remoteAddr;
addr.sin_port = htons(CONNECTPORT);
int res = connect(self.sock, (struct sockaddr *) &addr, sizeof(addr));
if (res == -1) {
close(self.sock);
NSLog(@"??????????connect error");
return NO;
}
NSLog(@"??????????socket connect to server success");
return YES;
}
- (BOOL)bind{
struct sockaddr_in client;
client.sin_family = AF_INET;
NSString *ipStr = [self ip];
if (ipStr.length <= 0) {
return NO;
}
const char *ip = [ipStr cStringUsingEncoding:NSASCIIStringEncoding];
client.sin_addr.s_addr = inet_addr(ip);
client.sin_port = htons(CONNECTPORT);
int bd = bind(self.sock, (struct sockaddr *) &client, sizeof(client));
if (bd == -1) {
close(self.sock);
NSLog(@"??????????bind error : %d",bd);
return NO;
}
return YES;
}
- (BOOL)listen{
int ls = listen(self.sock, 128);
if (ls == -1) {
close(self.sock);
NSLog(@"??????????listen error : %d",ls);
return NO;
}
return YES;
}
- (void)receive{
dispatch_async(dispatch_get_global_queue(0, 0), ^{
[self recvData];
});
}
- (NSString *)ip{
NSString *ip = nil;
struct ifaddrs *addrs = NULL;
struct ifaddrs *tmpAddrs = NULL;
BOOL res = getifaddrs(&addrs);
if (res == 0) {
tmpAddrs = addrs;
while (tmpAddrs != NULL) {
if(tmpAddrs->ifa_addr->sa_family == AF_INET) {
// Check if interface is en0 which is the wifi connection on the iPhone
NSLog(@"%@",[NSString stringWithUTF8String:inet_ntoa(((struct sockaddr_in *)tmpAddrs->ifa_addr)->sin_addr)]);
if([[NSString stringWithUTF8String:tmpAddrs->ifa_name] isEqualToString:@"en0"]) {
// Get NSString from C String
ip = [NSString stringWithUTF8String:inet_ntoa(((struct sockaddr_in *)tmpAddrs->ifa_addr)->sin_addr)];
}
}
tmpAddrs = tmpAddrs->ifa_next;
}
}
// Free memory
freeifaddrs(addrs);
NSLog(@"??????????%@",ip);
return ip;
}
-(void)close{
int res = close(self.sock);
NSLog(@"??????????shut down : %d",res);
}
- (void)recvData{
}
-(void)dealloc{
[self.recvThread stop];
}
@end
我創(chuàng)建了一個(gè) socket 的父類，然后 server 和 client 分別繼承這個(gè)類舍沙，來(lái)實(shí)現(xiàn)近上，鏈接綁定等操作，可以看到有很多數(shù)據(jù)可以設(shè)置拂铡，有些可以不用壹无，這里不是核心，核心是怎樣收發(fā)數(shù)據(jù)
發(fā)送屏幕貢共享數(shù)據(jù)
//
// RongRTCClientSocket.m
// SealRTC
//
// Created by 孫承秀 on 2020/5/7.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "RongRTCClientSocket.h"
#import <arpa/inet.h>
#import <netdb.h>
#import <sys/types.h>
#import <sys/socket.h>
#import <ifaddrs.h>
#import "RongRTCThread.h"
#import "RongRTCSocketHeader.h"
#import "RongRTCVideoEncoder.h"
@interface RongRTCClientSocket()<RongRTCCodecProtocol>{
pthread_mutex_t lock;
}
/**
video encoder
*/
@property(nonatomic , strong)RongRTCVideoEncoder *encoder;
/**
encode queue
*/
@property(nonatomic , strong)dispatch_queue_t encodeQueue;
@end
@implementation RongRTCClientSocket
- (BOOL)createCliectSocket{
if ([self createSocket] == -1) {
return NO;
}
BOOL isC = [self connect];
[self setSendBuffer];
[self setSendingTimeout];
if (isC) {
_encodeQueue = dispatch_queue_create("com.rongcloud.encodequeue", NULL);
[self createVideoEncoder];
return YES;
} else {
return NO;
}
}
- (void)createVideoEncoder{
self.encoder = [[RongRTCVideoEncoder alloc] init];
self.encoder.delegate = self;
RongRTCVideoEncoderSettings *settings = [[RongRTCVideoEncoderSettings alloc] init];
settings.width = 720;
settings.height = 1280;
settings.startBitrate = 300;
settings.maxFramerate = 30;
settings.minBitrate = 1000;
[self.encoder configWithSettings:settings onQueue:_encodeQueue];
}
-(void)cliectSend:(NSData *)data{
//data length
NSUInteger dataLength = data.length;
// data header struct
DataHeader dataH;
memset((void *)&dataH, 0, sizeof(dataH));
// pre
PreHeader preH;
memset((void *)&preH, 0, sizeof(preH));
preH.pre[0] = '&';
preH.dataLength = dataLength;
dataH.preH = preH;
// buffer
int headerlength = sizeof(dataH);
int totalLength = dataLength + headerlength;
// srcbuffer
Byte *src = (Byte *)[data bytes];
// send buffer
char *buffer = (char *)malloc(totalLength * sizeof(char));
memcpy(buffer, &dataH, headerlength);
memcpy(buffer + headerlength, src, dataLength);
// tosend
[self sendBytes:buffer length:totalLength];
free(buffer);
}
- (void)encodeBuffer:(CMSampleBufferRef)sampleBuffer{
[self.encoder encode:sampleBuffer];
}
- (void)sendBytes:(char *)bytes length:(int )length {
LOCK(self->lock);
int hasSendLength = 0;
while (hasSendLength < length) {
// connect socket success
if (self.sock > 0) {
// send
int sendRes = send(self.sock, bytes, length - hasSendLength, 0);
if (sendRes == -1 || sendRes == 0) {
UNLOCK(self->lock);
NSLog(@"??????????send buffer error");
[self close];
break;
}
hasSendLength += sendRes;
bytes += sendRes;
} else {
NSLog(@"??????????client socket connect error");
UNLOCK(self->lock);
}
}
UNLOCK(self->lock);
}
-(void)spsData:(NSData *)sps ppsData:(NSData *)pps{
[self cliectSend:sps];
[self cliectSend:pps];
}
-(void)naluData:(NSData *)naluData{
[self cliectSend:naluData];
}
-(void)dealloc{
NSLog(@"??????????dealoc cliect socket");
}
@end
這里核心思想是拿到我們屏幕共享的數(shù)據(jù)之后感帅，要先經(jīng)過(guò)壓縮斗锭，壓縮完成，會(huì)通過(guò)回調(diào)失球，會(huì)給我們當(dāng)前類岖是，然后通過(guò) cliectSend方法，發(fā)給主app实苞，我這里是自定義了一個(gè)頭部豺撑，頭部添加了一個(gè)前綴和一個(gè)每次發(fā)送字節(jié)的長(zhǎng)度，然后接收端去解析這個(gè)數(shù)據(jù)就行黔牵，核心都在這里
-(void)cliectSend:(NSData *)data{
//data length
NSUInteger dataLength = data.length;
// data header struct
DataHeader dataH;
memset((void *)&dataH, 0, sizeof(dataH));
// pre
PreHeader preH;
memset((void *)&preH, 0, sizeof(preH));
preH.pre[0] = '&';
preH.dataLength = dataLength;
dataH.preH = preH;
// buffer
int headerlength = sizeof(dataH);
int totalLength = dataLength + headerlength;
// srcbuffer
Byte *src = (Byte *)[data bytes];
// send buffer
char *buffer = (char *)malloc(totalLength * sizeof(char));
memcpy(buffer, &dataH, headerlength);
memcpy(buffer + headerlength, src, dataLength);
// tosend
[self sendBytes:buffer length:totalLength];
free(buffer);
}
大家仔細(xì)理解一下聪轿。
接收屏幕共享數(shù)據(jù)
//
// RongRTCServerSocket.m
// SealRTC
//
// Created by 孫承秀 on 2020/5/7.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "RongRTCServerSocket.h"
#import <arpa/inet.h>
#import <netdb.h>
#import <sys/types.h>
#import <sys/socket.h>
#import <ifaddrs.h>
#import <UIKit/UIKit.h>
#import "RongRTCThread.h"
#import "RongRTCSocketHeader.h"
#import "RongRTCVideoDecoder.h"
@interface RongRTCServerSocket()<RongRTCCodecProtocol>
{
pthread_mutex_t lock;
int _frameTime;
CMTime _lastPresentationTime;
Float64 _currentMediaTime;
Float64 _currentVideoTime;
dispatch_queue_t _frameQueue;
}
@property (nonatomic, assign) int acceptSock;
/**
data length
*/
@property(nonatomic , assign)NSUInteger dataLength;
/**
timeData
*/
@property(nonatomic , strong)NSData *timeData;
/**
decoder queue
*/
@property(nonatomic , strong)dispatch_queue_t decoderQueue;
/**
decoder
*/
@property(nonatomic , strong)RongRTCVideoDecoder *decoder;
@end
@implementation RongRTCServerSocket
- (BOOL)createServerSocket{
if ([self createSocket] == -1) {
return NO;
}
[self setRecvBuffer];
[self setRecvTimeout];
BOOL isB = [self bind];
BOOL isL = [self listen];
if (isB && isL) {
_decoderQueue = dispatch_queue_create("com.rongcloud.decoderQueue", NULL);
_frameTime = 0;
[self createDecoder];
[self receive];
return YES;
} else {
return NO;
}
}
- (void)createDecoder{
self.decoder = [[RongRTCVideoDecoder alloc] init];
self.decoder.delegate = self;
RongRTCVideoEncoderSettings *settings = [[RongRTCVideoEncoderSettings alloc] init];
settings.width = 720;
settings.height = 1280;
settings.startBitrate = 300;
settings.maxFramerate = 30;
settings.minBitrate = 1000;
[self.decoder configWithSettings:settings onQueue:_decoderQueue];
}
-(void)recvData{
struct sockaddr_in rest;
socklen_t rest_size = sizeof(struct sockaddr_in);
self.acceptSock = accept(self.sock, (struct sockaddr *) &rest, &rest_size);
while (self.acceptSock != -1) {
DataHeader dataH;
memset(&dataH, 0, sizeof(dataH));
if (![self receveData:(char *)&dataH length:sizeof(dataH)]) {
continue;
}
PreHeader preH = dataH.preH;
char pre = preH.pre[0];
if (pre == '&') {
// rongcloud socket
NSUInteger dataLenght = preH.dataLength;
char *buff = (char *)malloc(sizeof(char) * dataLenght);
if ([self receveData:(char *)buff length:dataLenght]) {
NSData *data = [NSData dataWithBytes:buff length:dataLenght];
[self.decoder decode:data];
free(buff);
}
} else {
NSLog(@"??????????pre is not &");
return;
}
}
}
- (BOOL)receveData:(char *)data length:(NSUInteger)length{
LOCK(lock);
int recvLength = 0;
while (recvLength < length) {
ssize_t res = recv(self.acceptSock, data, length - recvLength, 0);
if (res == -1 || res == 0) {
UNLOCK(lock);
NSLog(@"??????????recv data error");
break;
}
recvLength += res;
data += res;
}
UNLOCK(lock);
return YES;
}
-(void)didGetDecodeBuffer:(CVPixelBufferRef)pixelBuffer {
_frameTime += 1000;
CMTime pts = CMTimeMake(_frameTime, 1000);
CMSampleBufferRef sampleBuffer = [RongRTCBufferUtil sampleBufferFromPixbuffer:pixelBuffer time:pts];
// 查看解碼數(shù)據(jù)是否有問(wèn)題，如果image能顯示猾浦，就說(shuō)明對(duì)了陆错。
// 通過(guò)打斷點(diǎn) 將鼠標(biāo)放在 iamge 腦袋上，就可以看到數(shù)據(jù)了金赦，點(diǎn)擊那個(gè)小眼睛
UIImage *image = [RongRTCBufferUtil imageFromBuffer:sampleBuffer];
[self.delegate didProcessSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
-(void)close{
int res = close(self.acceptSock);
self.acceptSock = -1;
NSLog(@"??????????shut down server: %d",res);
[super close];
}
-(void)dealloc{
NSLog(@"??????????dealoc server socket");
}
@end
這里音瓷，通過(guò) socket 收到數(shù)據(jù)之后，會(huì)循環(huán)一直收數(shù)據(jù)夹抗，然后進(jìn)行解碼绳慎，最后通過(guò) 代理 didGetDecodeBuffer 回調(diào)數(shù)據(jù)，然后再拋出代理給app層，通過(guò)第三方SDK發(fā)送偷线，就可以了
3. videotoolbox 硬編碼
//
// RongRTCVideoEncoder.m
// SealRTC
//
// Created by 孫承秀 on 2020/5/13.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "RongRTCVideoEncoder.h"
#import "helpers.h"
@interface RongRTCVideoEncoder(){
VTCompressionSessionRef _compressionSession;
int _frameTime;
}
/**
settings
*/
@property(nonatomic , strong )RongRTCVideoEncoderSettings *settings;
/**
callback queue
*/
@property(nonatomic , strong )dispatch_queue_t callbackQueue;
- (void)sendSpsAndPPSWithSampleBuffer:(CMSampleBufferRef)sampleBuffer;
- (void)sendNaluData:(CMSampleBufferRef)sampleBuffer;
@end
void compressionOutputCallback(void *encoder,
void *params,
OSStatus status,
VTEncodeInfoFlags infoFlags,
CMSampleBufferRef sampleBuffer){
RongRTCVideoEncoder *videoEncoder = (__bridge RongRTCVideoEncoder *)encoder;
if (status != noErr) {
return;
}
if (infoFlags & kVTEncodeInfo_FrameDropped) {
return;
}
BOOL isKeyFrame = NO;
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0);
if (attachments != nullptr && CFArrayGetCount(attachments)) {
CFDictionaryRef attachment = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)) ;
isKeyFrame = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
}
CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(sampleBuffer);
CMBlockBufferRef contiguous_buffer = nullptr;
if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) {
status = CMBlockBufferCreateContiguous(
nullptr, block_buffer, nullptr, nullptr, 0, 0, 0, &contiguous_buffer);
if (status != noErr) {
return;
}
} else {
contiguous_buffer = block_buffer;
CFRetain(contiguous_buffer);
block_buffer = nullptr;
}
size_t block_buffer_size = CMBlockBufferGetDataLength(contiguous_buffer);
if (isKeyFrame) {
[videoEncoder sendSpsAndPPSWithSampleBuffer:sampleBuffer];
}
if (contiguous_buffer) {
CFRelease(contiguous_buffer);
}
[videoEncoder sendNaluData:sampleBuffer];
}
@implementation RongRTCVideoEncoder
@synthesize settings = _settings;
@synthesize callbackQueue = _callbackQueue;
- (BOOL)configWithSettings:(RongRTCVideoEncoderSettings *)settings onQueue:(nonnull dispatch_queue_t)queue{
self.settings = settings;
if (queue) {
_callbackQueue = queue;
} else {
_callbackQueue = dispatch_get_main_queue();
}
if ([self resetCompressionSession:settings]) {
_frameTime = 0;
return YES;
} else {
return NO;
}
}
- (BOOL)resetCompressionSession:(RongRTCVideoEncoderSettings *)settings {
[self destroyCompressionSession];
OSStatus status = VTCompressionSessionCreate(nullptr, settings.width, settings.height, kCMVideoCodecType_H264, nullptr, nullptr, nullptr, compressionOutputCallback, (__bridge void * _Nullable)(self), &_compressionSession);
if (status != noErr) {
return NO;
}
[self configureCompressionSession:settings];
return YES;
}
- (void)configureCompressionSession:(RongRTCVideoEncoderSettings *)settings{
if (_compressionSession) {
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true);
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 10);
uint32_t targetBps = settings.startBitrate * 1000;
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, targetBps);
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, settings.maxFramerate);
int bitRate = settings.width * settings.height * 3 * 4 * 4;
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitRate);
int bitRateLimit = settings.width * settings.height * 3 * 4;
SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_DataRateLimits, bitRateLimit);
}
}
-(void)encode:(CMSampleBufferRef)sampleBuffer{
// CFRetain(sampleBuffer);
// dispatch_async(_encodeQueue, ^{
CVImageBufferRef imageBuffer = (CVImageBufferRef)CMSampleBufferGetImageBuffer(sampleBuffer);
CMTime pts = CMTimeMake(self->_frameTime++, 1000);
VTEncodeInfoFlags flags;
OSStatus res = VTCompressionSessionEncodeFrame(self->_compressionSession,
imageBuffer,
pts,
kCMTimeInvalid,
NULL, NULL, &flags);
// CFRelease(sampleBuffer);
if (res != noErr) {
NSLog(@"encode frame error:%d", (int)res);
VTCompressionSessionInvalidate(self->_compressionSession);
CFRelease(self->_compressionSession);
self->_compressionSession = NULL;
return;
}
// });
}
- (void)sendSpsAndPPSWithSampleBuffer:(CMSampleBufferRef)sampleBuffer{
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
const uint8_t *sps ;
const uint8_t *pps;
size_t spsSize ,ppsSize , spsCount,ppsCount;
OSStatus spsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &sps, &spsSize, &spsCount, NULL);
OSStatus ppsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 1, &pps, &ppsSize, &ppsCount, NULL);
if (spsStatus == noErr && ppsStatus == noErr) {
const char bytes[] = "\x00\x00\x00\x01";
size_t length = (sizeof bytes) - 1;
NSMutableData *spsData = [NSMutableData dataWithCapacity:4+ spsSize];
NSMutableData *ppsData = [NSMutableData dataWithCapacity:4 + ppsSize];
[spsData appendBytes:bytes length:length];
[spsData appendBytes:sps length:spsSize];
[ppsData appendBytes:bytes length:length];
[ppsData appendBytes:pps length:ppsSize];
if (self && self.callbackQueue) {
dispatch_async(self.callbackQueue, ^{
if (self.delegate && [self.delegate respondsToSelector:@selector(spsData:ppsData:)]) {
[self.delegate spsData:spsData ppsData:ppsData];
}
});
}
} else {
NSLog(@"?? sps status:%@,pps status:%@",@(spsStatus),@(ppsStatus));
}
}
- (void)sendNaluData:(CMSampleBufferRef)sampleBuffer{
size_t totalLength = 0;
size_t lengthAtOffset=0;
char *dataPointer;
CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
OSStatus status1 = CMBlockBufferGetDataPointer(blockBuffer, 0, &lengthAtOffset, &totalLength, &dataPointer);
if (status1 != noErr) {
NSLog(@"video encoder error, status = %d", (int)status1);
return;
}
static const int h264HeaderLength = 4;
size_t bufferOffset = 0;
while (bufferOffset < totalLength - h264HeaderLength) {
uint32_t naluLength = 0;
memcpy(&naluLength, dataPointer + bufferOffset, h264HeaderLength);
naluLength = CFSwapInt32BigToHost(naluLength);
const char bytes[] = "\x00\x00\x00\x01";
NSMutableData *naluData = [NSMutableData dataWithCapacity:4 + naluLength];
[naluData appendBytes:bytes length:4];
[naluData appendBytes:dataPointer + bufferOffset + h264HeaderLength length:naluLength];
dispatch_async(self.callbackQueue, ^{
if (self.delegate && [self.delegate respondsToSelector:@selector(naluData:)]) {
[self.delegate naluData:naluData];
}
});
bufferOffset += naluLength + h264HeaderLength;
}
}
- (void)destroyCompressionSession{
if (_compressionSession) {
VTCompressionSessionInvalidate(_compressionSession);
CFRelease(_compressionSession);
_compressionSession = nullptr;
}
}
- (void)dealloc
{
if (_compressionSession) {
VTCompressionSessionCompleteFrames(_compressionSession, kCMTimeInvalid);
VTCompressionSessionInvalidate(_compressionSession);
CFRelease(_compressionSession);
_compressionSession = NULL;
}
}
@end
4. videotoolbox 解碼
//
// RongRTCVideoDecoder.m
// SealRTC
//
// Created by 孫承秀 on 2020/5/14.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "RongRTCVideoDecoder.h"
#import <UIKit/UIKit.h>
#import "helpers.h"
@interface RongRTCVideoDecoder(){
uint8_t *_sps;
NSUInteger _spsSize;
uint8_t *_pps;
NSUInteger _ppsSize;
CMVideoFormatDescriptionRef _videoFormatDescription;
VTDecompressionSessionRef _decompressionSession;
}
/**
settings
*/
@property(nonatomic , strong )RongRTCVideoEncoderSettings *settings;
/**
callback queue
*/
@property(nonatomic , strong )dispatch_queue_t callbackQueue;
@end
void DecoderOutputCallback(void * CM_NULLABLE decompressionOutputRefCon,
void * CM_NULLABLE sourceFrameRefCon,
OSStatus status,
VTDecodeInfoFlags infoFlags,
CM_NULLABLE CVImageBufferRef imageBuffer,
CMTime presentationTimeStamp,
CMTime presentationDuration ) {
if (status != noErr) {
NSLog(@"?? decoder callback error :%@", @(status));
return;
}
CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *)sourceFrameRefCon;
*outputPixelBuffer = CVPixelBufferRetain(imageBuffer);
RongRTCVideoDecoder *decoder = (__bridge RongRTCVideoDecoder *)(decompressionOutputRefCon);
dispatch_async(decoder.callbackQueue, ^{
[decoder.delegate didGetDecodeBuffer:imageBuffer];
CVPixelBufferRelease(imageBuffer);
});
}
@implementation RongRTCVideoDecoder
@synthesize settings = _settings;
@synthesize callbackQueue = _callbackQueue;
-(BOOL)configWithSettings:(RongRTCVideoEncoderSettings *)settings onQueue:(dispatch_queue_t)queue{
self.settings = settings;
if (queue) {
_callbackQueue = queue;
} else {
_callbackQueue = dispatch_get_main_queue();
}
return YES;
}
- (BOOL)createVT{
if (_decompressionSession) {
return YES;
}
const uint8_t * const parameterSetPointers[2] = {_sps, _pps};
const size_t parameterSetSizes[2] = {_spsSize, _ppsSize};
int naluHeaderLen = 4;
OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2, parameterSetPointers, parameterSetSizes, naluHeaderLen, &_videoFormatDescription );
if (status != noErr) {
NSLog(@"??????????CMVideoFormatDescriptionCreateFromH264ParameterSets error:%@", @(status));
return false;
}
NSDictionary *destinationImageBufferAttributes =
@{
(id)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange],
(id)kCVPixelBufferWidthKey: [NSNumber numberWithInteger:self.settings.width],
(id)kCVPixelBufferHeightKey: [NSNumber numberWithInteger:self.settings.height],
(id)kCVPixelBufferOpenGLCompatibilityKey: [NSNumber numberWithBool:true]
};
VTDecompressionOutputCallbackRecord CallBack;
CallBack.decompressionOutputCallback = DecoderOutputCallback;
CallBack.decompressionOutputRefCon = (__bridge void * _Nullable)(self);
status = VTDecompressionSessionCreate(kCFAllocatorDefault, _videoFormatDescription, NULL, (__bridge CFDictionaryRef _Nullable)(destinationImageBufferAttributes), &CallBack, &_decompressionSession);
if (status != noErr) {
NSLog(@"??????????VTDecompressionSessionCreate error:%@", @(status));
return false;
}
status = VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime,kCFBooleanTrue);
return YES;
}
- (CVPixelBufferRef)decode:(uint8_t *)frame withSize:(uint32_t)frameSize {
CVPixelBufferRef outputPixelBuffer = NULL;
CMBlockBufferRef blockBuffer = NULL;
CMBlockBufferFlags flag0 = 0;
OSStatus status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, frame, frameSize, kCFAllocatorNull, NULL, 0, frameSize, flag0, &blockBuffer);
if (status != kCMBlockBufferNoErr) {
NSLog(@"??????????VCMBlockBufferCreateWithMemoryBlock code=%d", (int)status);
CFRelease(blockBuffer);
return outputPixelBuffer;
}
CMSampleBufferRef sampleBuffer = NULL;
const size_t sampleSizeArray[] = {frameSize};
status = CMSampleBufferCreateReady(kCFAllocatorDefault, blockBuffer, _videoFormatDescription, 1, 0, NULL, 1, sampleSizeArray, &sampleBuffer);
if (status != noErr || !sampleBuffer) {
NSLog(@"??????????CMSampleBufferCreateReady failed status=%d", (int)status);
CFRelease(blockBuffer);
return outputPixelBuffer;
}
VTDecodeFrameFlags flag1 = kVTDecodeFrame_1xRealTimePlayback;
VTDecodeInfoFlags infoFlag = kVTDecodeInfo_Asynchronous;
status = VTDecompressionSessionDecodeFrame(_decompressionSession, sampleBuffer, flag1, &outputPixelBuffer, &infoFlag);
if (status == kVTInvalidSessionErr) {
NSLog(@"??????????decode frame error with session err status =%d", (int)status);
[self resetVT];
} else {
if (status != noErr) {
NSLog(@"??????????decode frame error with status =%d", (int)status);
}
}
CFRelease(sampleBuffer);
CFRelease(blockBuffer);
return outputPixelBuffer;
}
- (void)resetVT{
[self destorySession];
[self createVT];
}
-(void)decode:(NSData *)data{
// dispatch_async(_callbackQueue, ^{
uint8_t *frame = (uint8_t*)[data bytes];
uint32_t length = data.length;
uint32_t nalSize = (uint32_t)(length - 4);
uint32_t *pNalSize = (uint32_t *)frame;
*pNalSize = CFSwapInt32HostToBig(nalSize);
int type = (frame[4] & 0x1F);
CVPixelBufferRef pixelBuffer = NULL;
switch (type) {
case 0x05:
if ([self createVT]) {
pixelBuffer= [self decode:frame withSize:length];
}
break;
case 0x07:
self->_spsSize = length - 4;
self->_sps = (uint8_t *)malloc(self->_spsSize);
memcpy(self->_sps, &frame[4], self->_spsSize);
break;
case 0x08:
self->_ppsSize = length - 4;
self->_pps = (uint8_t *)malloc(self->_ppsSize);
memcpy(self->_pps, &frame[4], self->_ppsSize);
break;
default:
if ([self createVT]) {
pixelBuffer = [self decode:frame withSize:length];
}
break;
}
// });
}
- (void)dealloc
{
[self destorySession];
}
- (void)destorySession{
if (_decompressionSession) {
VTDecompressionSessionInvalidate(_decompressionSession);
CFRelease(_decompressionSession);
_decompressionSession = NULL;
}
}
@end
5. 工具類
//
// RongRTCBufferUtil.m
// SealRTC
//
// Created by 孫承秀 on 2020/5/8.
// Copyright ? 2020 RongCloud. All rights reserved.
//
#import "RongRTCBufferUtil.h"
/// 下面的這些方法，一定要記得release沽甥，有的沒(méi)有在方法里面release声邦，但是在外面release了，要不然會(huì)內(nèi)存泄漏
@implementation RongRTCBufferUtil
+ (UIImage *)imageFromBuffer:(CMSampleBufferRef)buffer {
CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)CMSampleBufferGetImageBuffer(buffer);
CIImage *ciImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
CIContext *temporaryContext = [CIContext contextWithOptions:nil];
CGImageRef videoImage = [temporaryContext createCGImage:ciImage fromRect:CGRectMake(0, 0, CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer))];
UIImage *image = [UIImage imageWithCGImage:videoImage];
CGImageRelease(videoImage);
return image;
}
+ (UIImage *)compressImage:(UIImage *)image newWidth:(CGFloat)newImageWidth
{
if (!image) return nil;
float imageWidth = image.size.width;
float imageHeight = image.size.height;
float width = newImageWidth;
float height = image.size.height/(image.size.width/width);
float widthScale = imageWidth /width;
float heightScale = imageHeight /height;
UIGraphicsBeginImageContext(CGSizeMake(width, height));
if (widthScale > heightScale) {
[image drawInRect:CGRectMake(0, 0, imageWidth /heightScale , height)];
}
else {
[image drawInRect:CGRectMake(0, 0, width , imageHeight /widthScale)];
}
UIImage *newImage = UIGraphicsGetImageFromCurrentImageContext();
UIGraphicsEndImageContext();
return newImage;
}
+(CVPixelBufferRef)CVPixelBufferRefFromUiImage:(UIImage *)img {
CGSize size = img.size;
CGImageRef image = [img CGImage];
NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys:
[NSNumber numberWithBool:YES], kCVPixelBufferCGImageCompatibilityKey,
[NSNumber numberWithBool:YES], kCVPixelBufferCGBitmapContextCompatibilityKey, nil];
CVPixelBufferRef pxbuffer = NULL;
CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, size.width, size.height, kCVPixelFormatType_32ARGB, (__bridge CFDictionaryRef) options, &pxbuffer);
NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL);
CVPixelBufferLockBaseAddress(pxbuffer, 0);
void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer);
NSParameterAssert(pxdata != NULL);
CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();
CGContextRef context = CGBitmapContextCreate(pxdata, size.width, size.height, 8, 4*size.width, rgbColorSpace, kCGImageAlphaPremultipliedFirst);
NSParameterAssert(context);
CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image), CGImageGetHeight(image)), image);
CGColorSpaceRelease(rgbColorSpace);
CGContextRelease(context);
CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
return pxbuffer;
}
+ (CMSampleBufferRef)sampleBufferFromPixbuffer:(CVPixelBufferRef)pixbuffer time:(CMTime)time{
CMSampleBufferRef sampleBuffer = NULL;
// //獲取視頻信息
CMVideoFormatDescriptionRef videoInfo = NULL;
OSStatus result = CMVideoFormatDescriptionCreateForImageBuffer(NULL, pixbuffer, &videoInfo);
CMTime currentTime = time;
// CMSampleTimingInfo timing = {currentTime, currentTime, kCMTimeInvalid};
CMSampleTimingInfo timing = {currentTime, currentTime, kCMTimeInvalid};
result = CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault,pixbuffer, true, NULL, NULL, videoInfo, &timing, &sampleBuffer);
CFRelease(videoInfo);
return sampleBuffer;
}
+ (size_t)getCMTimeSize{
size_t size = sizeof(CMTime);
return size;
}
@end
這個(gè)類主要是 cpu 級(jí)別的摆舟，CMSampleBufferRef 轉(zhuǎn) UIImage亥曹，UIImage 轉(zhuǎn) CVPixelBufferRef， CVPixelBufferRef 轉(zhuǎn) CMSampleBufferRef恨诱，還有裁剪圖片媳瞪，注意這里有的沒(méi)有release是我在外面release了，一定要注意內(nèi)存泄漏問(wèn)題照宝，要不然你的app會(huì)內(nèi)存暴漲蛇受。
總結(jié)
糖果的坑：
- 這里可能都是貼的代碼，文字很少厕鹃，時(shí)間緊迫兢仰，給大家提供思路和我經(jīng)歷的坑就好了，一開(kāi)始做這個(gè)的時(shí)候剂碴，沒(méi)有使用 videotoolbox把将，使用cpu對(duì)bugger進(jìn)行處理，軟編軟解忆矛，其實(shí)也是通過(guò)socket發(fā)送出去察蹲，但是發(fā)現(xiàn)，extension 屏幕是有內(nèi)存限制的催训，最大50M洽议，在extension 我通過(guò)裁剪和壓縮的代碼，發(fā)現(xiàn)經(jīng)常會(huì)崩潰漫拭，超過(guò)50M绞铃，程序被殺死，然后每次壓縮的數(shù)據(jù)其實(shí)也很大嫂侍，效果很不好儿捧，后來(lái)想到了用蘋(píng)果的 videotoolbox。
- videotoolbox 后臺(tái)解碼一直失敗挑宠，肯定不行的菲盾，屏幕共享是必須要在后臺(tái)可以錄制的，經(jīng)過(guò) google 之后各淀，發(fā)現(xiàn)懒鉴，把videotoolbox 重啟一下就可以了，在我的代碼里面有體現(xiàn)
- 解碼成功，但是通過(guò)融云的庫(kù)發(fā)出去临谱，幀率很低不連貫璃俗，圖片都是有的，而且要是渲染也是沒(méi)有問(wèn)題的悉默，但是通過(guò)我們?nèi)谠频膚ebrtc發(fā)送的話城豁，發(fā)現(xiàn)幀率為0或者1然后就開(kāi)始改pts，想過(guò)用我們?nèi)谠频腟DK采集的攝像頭的pts發(fā)現(xiàn)可以抄课，但是唱星，有個(gè)問(wèn)題，開(kāi)發(fā)者不可能一直這么用跟磨，最后經(jīng)過(guò)改造之后间聊，終于可以了，這個(gè)坑抵拘，憋了我好幾天哎榴，終于在不依賴我們SDK的情況下，實(shí)現(xiàn)無(wú)縫抽出屏幕共享模塊僵蛛。
上面的代碼可能還有bug和問(wèn)題叹话，寫(xiě)到這里，demo已經(jīng)能看到效果了墩瞳，如果有什么bug或者問(wèn)題驼壶，你們給我留言我改下就可以了，但至少我覺(jué)得思路是正確沒(méi)有問(wèn)題的應(yīng)該喉酌。
上面的代碼在github可以下載热凹，要想看到效果，就在
-(void)didGetDecodeBuffer:(CVPixelBufferRef)pixelBuffer {
_frameTime += 1000;
CMTime pts = CMTimeMake(_frameTime, 1000);
CMSampleBufferRef sampleBuffer = [RongRTCBufferUtil sampleBufferFromPixbuffer:pixelBuffer time:pts];
// 查看解碼數(shù)據(jù)是否有問(wèn)題泪电，如果image能顯示般妙，就說(shuō)明對(duì)了。
// 通過(guò)打斷點(diǎn) 將鼠標(biāo)放在 iamge 腦袋上相速，就可以看到數(shù)據(jù)了碟渺，點(diǎn)擊那個(gè)小眼睛
UIImage *image = [RongRTCBufferUtil imageFromBuffer:sampleBuffer];
[self.delegate didProcessSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
這個(gè)方法的image下面，打一個(gè)斷點(diǎn)突诬，鼠標(biāo)放在 image上面苫拍，然后點(diǎn)擊小眼睛，就可以看到extension發(fā)過(guò)來(lái)的每一幀圖片數(shù)據(jù)了旺隙。
