播放器具備的功能
- 同時(shí)播放音視頻
- 單獨(dú)播放音頻堂氯，視頻
- 開(kāi)始，暫停左电，停止
- 靜音廉侧，音量控制
- 拖動(dòng)進(jìn)度
播放流程
- 解封裝
- 初始化音頻信息
- 初始化視頻信息
- 開(kāi)啟音頻子線(xiàn)程程，開(kāi)始音頻拉取任務(wù)
- 開(kāi)啟視頻解碼子線(xiàn)程篓足，在這個(gè)子線(xiàn)程中while循環(huán)會(huì)一直讀取videoPackList中的頭元素段誊，有就進(jìn)行解碼，解碼完成栈拖，通知外部進(jìn)行渲染连舍；list為空，則直接continue
- while讀取AVPacket辱魁，放入對(duì)應(yīng)的數(shù)據(jù)緩存中
播放流程
解封裝
- 創(chuàng)建解封裝上下文
- 檢索流信息 (流相關(guān)的音頻和視頻數(shù)據(jù)都將會(huì)在這個(gè)
fmtCtx中) - 打印流信息到控制臺(tái)
// 創(chuàng)建解封裝上下文
ret = avformat_open_input(&fmtCtx, filename, nullptr, nullptr);
END(avformat_open_input);
// 檢索流信息
ret = avformat_find_stream_info(fmtCtx, nullptr);
END(avformat_find_stream_info);
// 打印流信息到控制臺(tái)
av_dump_format(fmtCtx, 0, filename, 0);
fflush(stderr);
初始化解碼器
- 根據(jù)type尋找到最合適的流信息
- 獲取流
- 為當(dāng)前流找到合適的解碼器
- 初始化解碼器上下文
- 從流中拷貝參數(shù)到解碼器上下文中
- 打開(kāi)解碼器
int VideoPlayer::initDecoder(AVCodecContext **decodecCtx, AVStream**stream, AVMediaType type) {
// 根據(jù)type尋找到最合適的流信息
int ret = av_find_best_stream(fmtCtx, type, -1, -1, nullptr, 0);
RET(av_find_best_stream);
// 獲取流
int streamIdx = ret;
*stream = fmtCtx->streams[streamIdx];
if (!*stream) {
cout << "stream is empty" << endl;
return -1;
}
// 為當(dāng)前流找到合適的解碼器
AVCodec *decoder = avcodec_find_decoder((*stream)->codecpar->codec_id);
if (!decoder) {
cout << "avcodec_find_decoder is empty" << endl;
return -1;
}
// 初始化解碼器上下文
*decodecCtx = avcodec_alloc_context3(decoder);
if (!decodecCtx) {
cout << "avcodec_alloc_context3 error" << endl;
}
// 從流中拷貝參數(shù)到解碼器上下文中
ret = avcodec_parameters_to_context(*decodecCtx, (*stream)->codecpar);
RET(avcodec_parameters_to_context);
// 打開(kāi)解碼器
ret = avcodec_open2(*decodecCtx, decoder, nullptr);
RET(avcodec_open2);
return 0;
}
初始化音頻信息
- 初始化解碼器
- 初始化音頻重采樣
- 初始化SDL
int VideoPlayer::initAudioInfo() {
// 初始化解碼器
int ret = initDecoder(&aDecodeCtx, &aStream, AVMEDIA_TYPE_AUDIO);
RET(initDecoder);
// 初始化音頻重采樣
ret = initSwr();
RET(initSwr);
// 初始化SDL
ret = initSDL();
RET(initSDL);
return 0;
}
初始化音頻重采樣
- 設(shè)置重采樣輸入?yún)?shù)
- 設(shè)置重采樣輸出參數(shù)
- 創(chuàng)建重采樣上下文
- 初始化重采樣上下文
- 初始化重采樣的輸入frame
- 初始化重采樣的輸出frame
- 為aSwrOutFrame的data[0]分配內(nèi)存空間
int VideoPlayer::initSwr() {
// 設(shè)置重采樣輸入?yún)?shù)
aSwrInSpec.sampleFmt = aDecodeCtx->sample_fmt;
aSwrInSpec.sampleRate = aDecodeCtx->sample_rate;
aSwrInSpec.chLayout = (int)aDecodeCtx->channel_layout;
aSwrInSpec.chs = aDecodeCtx->channels;
// 設(shè)置重采樣輸出參數(shù)
aSwrOutSpec.sampleFmt = AV_SAMPLE_FMT_S16;
aSwrOutSpec.sampleRate = 44100;
aSwrOutSpec.chLayout = AV_CH_LAYOUT_STEREO;
aSwrOutSpec.chs = av_get_channel_layout_nb_channels(aSwrOutSpec.chLayout);
aSwrOutSpec.bytesPerSampleFrame = aSwrOutSpec.chs * av_get_bytes_per_sample(aSwrOutSpec.sampleFmt);
// 創(chuàng)建重采樣上下文
aSwrCtx = swr_alloc_set_opts(nullptr,
// 輸出參數(shù)
aSwrOutSpec.chLayout,
aSwrOutSpec.sampleFmt,
aSwrOutSpec.sampleRate,
// 輸入?yún)?shù)
aSwrInSpec.chLayout,
aSwrInSpec.sampleFmt,
aSwrInSpec.sampleRate,
0, nullptr);
if (!aSwrCtx) {
cout << "swr_alloc_set_opts error" << endl;
return -1;
}
// 初始化重采樣上下文
int ret = swr_init(aSwrCtx);
RET(swr_init);
// 初始化重采樣的輸入frame
aSwrOutFrame = av_frame_alloc();
if (!aSwrOutFrame) {
cout << "av_frame_alloc error" << endl;
return -1;
}
// 初始化重采樣的輸出frame
aSwrInFrame = av_frame_alloc();
if (!aSwrInFrame) {
cout << "av_frame_alloc error" << endl;
return -1;
}
// 為aSwrOutFrame的data[0]分配內(nèi)存空間
ret = av_samples_alloc(aSwrOutFrame->data,
aSwrOutFrame->linesize,
aSwrOutSpec.chs,
4096, aSwrOutSpec.sampleFmt, 1);
RET(av_samples_alloc)
return 0;
}
初始化SDL
- 設(shè)置音頻播放參數(shù)：采樣率吱肌，采樣格式，聲道數(shù)弓坞，音頻緩沖區(qū)的樣本數(shù)量
- 傳遞給回調(diào)的參數(shù)
- 設(shè)置音頻回調(diào)
- 打開(kāi)音頻設(shè)備
int VideoPlayer::initSDL() {
// 音頻參數(shù)
SDL_AudioSpec spec;
// 采樣率
spec.freq = aSwrOutSpec.sampleRate;
// 采樣格式
spec.format = AUDIO_S16LSB;
// 聲道數(shù)
spec.channels = aSwrOutSpec.chs;
// 音頻緩沖區(qū)的樣本數(shù)量
spec.samples = 512;
// 傳遞給回調(diào)的參數(shù)
spec.userdata = this;
// 回調(diào)
spec.callback = sdlAudioCallbackFunc;
// 打開(kāi)音頻設(shè)備
if (SDL_OpenAudio(&spec, nullptr)) {
cout << "SDL_OpenAudio error" << endl;
return -1;
}
return 0;
}
SDL回調(diào)
void VideoPlayer::sdlAudioCallbackFunc(void *userData, uint8_t *stream, int len) {
VideoPlayer *player = (VideoPlayer*)userData;
player->sdlAudioCallback(stream, len);
}
void VideoPlayer::sdlAudioCallback(uint8_t *stream, int len) {
// 清零（靜音）
SDL_memset(stream, 0, len);
// len: SDL音頻緩沖區(qū)剩余的大心Ｔ铩（還未填充的大小）
while (len > 0) {
if (state == Paused) {
break;
}
if (state == Stopped) {
aCanFree = true;
break;
}
// 說(shuō)明當(dāng)前PCM的數(shù)據(jù)已經(jīng)全部拷貝到SDL的音頻緩沖區(qū)了
// 需要解碼下一個(gè)pkt锻弓，獲取新的PCM數(shù)據(jù)
if (aSwrOutIdx >= aSwrOutSiize) {
// 新的PCM的大小
aSwrOutSiize = decodeAudio();
// 索引清0
aSwrOutIdx = 0;
// 沒(méi)有解碼出PCM數(shù)據(jù)砾赔，那就靜音處理
if (aSwrOutSiize <= 0) {
// 假定PCM的大小
aSwrOutSiize = 1024;
// 給PCM填充0（靜音）
memset(aSwrOutFrame->data[0], 0, aSwrOutSiize);
}
}
// 本次需要填充到stream中的PCM的數(shù)據(jù)大小
int fillLen = aSwrOutSiize - aSwrOutIdx;
fillLen = min(fillLen, len);
// 獲取當(dāng)前音量
int volumn = mute ? 0 : ((this->volumn * 1.0 / Max) * SDL_MIX_MAXVOLUME);
cout << "volumn" << volumn << endl;
// 填充SDL緩沖區(qū)
SDL_MixAudio(stream,
aSwrOutFrame->data[0] + aSwrOutIdx,
fillLen, volumn);
// 移動(dòng)偏移量
len -= fillLen;
stream += fillLen;
aSwrOutIdx += fillLen;
cout << "SDL_MixAudio fillLen:" << fillLen << " aSwrOutIdx: " << aSwrOutIdx << " aSwrOutSiize: " << aSwrOutSiize << " len: " << len << endl;
}
cout << "len <= 0" << endl;
}
音頻解碼模塊
- 取出音頻包
- 發(fā)送數(shù)據(jù)到解碼器
- 從解碼器中獲取解碼之后的數(shù)據(jù)到輸入frame
- 重采樣PCM
int VideoPlayer::decodeAudio() {
// 加鎖
aMutex.lock();
if (aPktList.empty()) {
aMutex.unlock();
return 0;
}
AVPacket pkt = aPktList.front();
aPktList.pop_front();
cout << "list cout: " << aPktList.size() << endl;
aMutex.unlock();
// 保存音頻時(shí)鐘
if (pkt.pts != AV_NOPTS_VALUE) {
aTime = av_q2d(aStream->time_base) * pkt.pts;
// 通知外界：播放時(shí)間點(diǎn)發(fā)生了改變
if (callback.timeChanged) {
callback.timeChanged(userData, this);
}
}
// 如果是視頻，不能在這個(gè)位置判斷（不能提前釋放pkt青灼，不然會(huì)導(dǎo)致B幀暴心。P幀解碼失敗，畫(huà)面直接撕裂）
// 發(fā)現(xiàn)音頻的時(shí)間是早于seektime的杂拨，直接丟棄
if (aSeekTime >= 0) {
if (aTime < aSeekTime) {
// 釋放pkt
av_packet_unref(&pkt);
return 0;
} else {
aSeekTime = -1;
}
}
// 發(fā)送數(shù)據(jù)到解碼器
int ret = avcodec_send_packet(aDecodeCtx, &pkt);
av_packet_unref(&pkt);
RET(avcodec_send_packet);
// 從解碼器中獲取解碼之后的數(shù)據(jù)到輸入frame
ret = avcodec_receive_frame(aDecodeCtx, aSwrInFrame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
return 0;
} else RET(avcodec_receive_frame);
// 重采樣輸出的樣本數(shù)
int outSamples = (int)av_rescale_rnd(aSwrOutSpec.sampleRate,
aSwrInFrame->nb_samples,
aSwrInSpec.sampleRate, AV_ROUND_UP);
// 由于解碼出來(lái)的PCM专普，跟SDL要求的PCM格式可能不一致
// 所以需要需要重采樣
ret = swr_convert(aSwrCtx,
aSwrOutFrame->data,
outSamples,
(const uint8_t**)aSwrInFrame->data,
aSwrInFrame->nb_samples);
RET(swr_convert)
return ret * aSwrOutSpec.bytesPerSampleFrame;
}
初始化視頻信息
- 初始化解碼器
- 初始化像素格式轉(zhuǎn)換
初始化像素格式轉(zhuǎn)換
- 設(shè)置輸出參數(shù)
- 初始化像素格式轉(zhuǎn)換的上下文
- 初始化像素格式轉(zhuǎn)換的輸入frame
- 初始化像素格式轉(zhuǎn)換的輸出frame
- 給輸出緩沖區(qū)_vSwsOutFrame的data分配內(nèi)存空間
int VideoPlayer::initSws() {
int inW = vDecodeCtx->width;
int inH = vDecodeCtx->height;
// 輸出參數(shù)
vSwsOutSpec.width = inW >> 4 << 4;
vSwsOutSpec.height = inH >> 4 << 4;
vSwsOutSpec.pixFmt = AV_PIX_FMT_RGB24;
vSwsOutSpec.size = av_image_get_buffer_size(vSwsOutSpec.pixFmt, vSwsOutSpec.width, vSwsOutSpec.height, 1);
// 初始化像素格式轉(zhuǎn)換的上下文
vSwsCtx = sws_getContext(inW,
inH,
vDecodeCtx->pix_fmt,
vSwsOutSpec.width,
vSwsOutSpec.height,
vSwsOutSpec.pixFmt,
SWS_BILINEAR, nullptr, nullptr, nullptr);
if (!vSwsCtx) {
return -1;
}
// 初始化像素格式轉(zhuǎn)換的輸入frame
vSwsInFrame = av_frame_alloc();
// 初始化像素格式轉(zhuǎn)換的輸出frame
vSwsOutframe = av_frame_alloc();
// 給輸出緩沖區(qū)_vSwsOutFrame的data分配內(nèi)存空間
int ret = av_image_alloc(vSwsOutframe->data, vSwsOutframe->linesize, vSwsOutSpec.width, vSwsOutSpec.height, vSwsOutSpec.pixFmt, 1);
RET(av_image_alloc);
return 0;
}
視頻解碼模塊
- 發(fā)送數(shù)據(jù)到解碼器
- 獲取解碼后的數(shù)據(jù)
- 像素格式裝換
- 對(duì)外回調(diào)視頻幀
void VideoPlayer::decodeVideo() {
while (true) {
// 如果是暫停，并且沒(méi)有Seek操作
if (state == Paused && vSeekTime == -1) {
continue;
}
if (state == Stopped) {
vCanFree = true;
break;
}
vMutex.lock();
if (vPktList.empty()) {
vMutex.unlock();
continue;
}
// 取出頭部的視頻包
AVPacket pkt = vPktList.front();
vPktList.pop_front();
vMutex.unlock();
// 視頻時(shí)鐘
if (pkt.dts != AV_NOPTS_VALUE) {
vTime = av_q2d(vStream->time_base) * pkt.dts;
}
cout << "vTime: " << vTime << " aTime: " << aTime << endl;
// 發(fā)送數(shù)據(jù)到解碼器
int ret = avcodec_send_packet(vDecodeCtx, &pkt);
// 釋放pkt
av_packet_unref(&pkt);
CONTINUE(avcodec_send_packet);
while (true) {
// 獲取解碼后的數(shù)據(jù)
ret = avcodec_receive_frame(vDecodeCtx, vSwsInFrame);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
} else BREAK(avcodec_receive_frame);
// 一定要在解碼成功后弹沽，再進(jìn)行下面的判斷
// 發(fā)現(xiàn)視頻的時(shí)間早于seektime的檀夹，直接丟棄
if (vSeekTime >= 0) {
if (vTime < vSeekTime) {
continue;
} else {
vSeekTime = -1;
}
}
// 像素格式裝換
sws_scale(vSwsCtx,
vSwsInFrame->data,
vSwsInFrame->linesize,
0,
vDecodeCtx->height,
vSwsOutframe->data,
vSwsOutframe->linesize);
if (hasAudio) {// 有音頻
// 如果視頻包過(guò)早被解碼出來(lái)筋粗，那就需要等待對(duì)應(yīng)的音頻時(shí)鐘到達(dá)
while (vTime > aTime && state == Playing) {
cout << "如果視頻包過(guò)早被解碼出來(lái)，那就需要等待對(duì)應(yīng)的音頻時(shí)鐘到達(dá)" << endl;
SDL_Delay(5);
}
} else {
// TODO
}
// 把像素格式轉(zhuǎn)換后的圖片數(shù)據(jù)炸渡，拷貝一份出來(lái)
uint8_t *data = (uint8_t*)av_malloc(vSwsOutSpec.size);
memcpy(data, vSwsOutframe->data[0], vSwsOutSpec.size);
// 回調(diào)給外部進(jìn)行渲染
cout << "渲染了一幀" << vSwsOutframe->pts << " 剩余包數(shù)量：" << vPktList.size() << endl;
if (callback.didDecodeVideoFrame) {
callback.didDecodeVideoFrame(this->userData, this, data, vSwsOutSpec);
} else {
delete data;
data = nullptr;
}
}
}
}
資源釋放模塊
- 釋放fmtCtx
- 釋放音頻資源
- 釋放視頻資源
void VideoPlayer::free() {
while (hasAudio && !aCanFree);
while (hasVideo && !vCanFree);
while (!fmtCtxCanFree);
avformat_close_input(&fmtCtx);
fmtCtxCanFree = false;
seekTime = -1;
freeAudio();
freeVideo();
}
void VideoPlayer::freeAudio() {
aTime = 0;
aSwrOutIdx = 0;
aSwrOutSiize = 0;
aStream = nullptr;
aCanFree = false;
aSeekTime = -1;
clearAudioPktList();
avcodec_free_context(&aDecodeCtx);
swr_free(&aSwrCtx);
av_frame_free(&aSwrInFrame);
if (aSwrOutFrame) {
av_freep(&aSwrOutFrame->data[0]);
av_frame_free(&aSwrOutFrame);
}
SDL_PauseAudio(1);
SDL_CloseAudio();
}
void VideoPlayer::freeVideo() {
clearVideoPktList();
avcodec_free_context(&vDecodeCtx);
av_frame_free(&vSwsInFrame);
if (vSwsOutframe) {
av_freep(&vSwsOutframe->data[0]);
av_frame_free(&vSwsOutframe);
}
sws_freeContext(vSwsCtx);
vSwsCtx = nullptr;
vStream = nullptr;
vTime = 0;
vCanFree = false;
vSeekTime = -1;
}
總體流程
void VideoPlayer::readFile() {
int ret = 0;
// 創(chuàng)建解封裝上下文
ret = avformat_open_input(&fmtCtx, filename, nullptr, nullptr);
END(avformat_open_input);
// 檢索流信息
ret = avformat_find_stream_info(fmtCtx, nullptr);
END(avformat_find_stream_info);
// 打印流信息到控制臺(tái)
av_dump_format(fmtCtx, 0, filename, 0);
fflush(stderr);
// 初始化音頻信息
hasAudio = initAudioInfo() >= 0;
// 初始化視頻信息
hasVideo = initVideoInfo() >= 0;
// 到此為止初始化完畢
cout << "初始化完畢" << endl;
setState(Playing);
// 音頻解碼子線(xiàn)程：開(kāi)始工作
SDL_PauseAudio(0);
// 視頻解碼子線(xiàn)程：開(kāi)始工作
thread([this]() {
decodeVideo();
}).detach();
// 從輸入文件中讀取數(shù)據(jù)
AVPacket pkt;
while (state != Stopped) {
// 處理Seek操作
if (seekTime >= 0) {
int streamIdx;
if (hasAudio) { // 優(yōu)先使用音頻流索引
streamIdx = aStream->index;
} else {
streamIdx = vStream->index;
}
AVRational timeBase = fmtCtx->streams[streamIdx]->time_base;
int64_t ts = seekTime / av_q2d(timeBase);
ret = av_seek_frame(fmtCtx, streamIdx, ts, AVSEEK_FLAG_BACKWARD);
if (ret < 0) { // seek失敗
seekTime = -1;
cout << "Seek 失敗" << seekTime << ts << streamIdx << endl;
} else {
cout << "Seek 成功" << seekTime << ts << streamIdx << endl;
vSeekTime = seekTime;
aSeekTime = seekTime;
seekTime = -1;
aTime = 0;
vTime = 0;
// 清空之前的讀取的數(shù)據(jù)包
clearAudioPktList();
clearVideoPktList();
}
}
int vSize = (int)vPktList.size();
int aSize = (int)aPktList.size();
if (vSize >= AUDIO_MAX_PKT_SIZE || aSize >= AUDIO_MAX_PKT_SIZE) {
continue;
}
ret = av_read_frame(fmtCtx, &pkt);
if (ret == 0) {
if (pkt.stream_index == aStream->index) {
addAudioPkt(pkt);
} else if (pkt.stream_index == vStream->index) {
addVideoPkt(pkt);
} else {
av_packet_unref(&pkt);
}
} else if (ret == AVERROR_EOF) { // 讀取到了文件的尾部
if (vSize == 0 && aSize == 0) {
// 說(shuō)明文件正常播放完畢
fmtCtxCanFree = true;
break;
}
} else {
ERROR_BUF;
continue;
}
}
if (fmtCtxCanFree) {
stop();
} else {
fmtCtxCanFree = true;
}
}
對(duì)外接口
void VideoPlayer::setFilename(string name) {
const char *filename = name.c_str();
memcpy(this->filename, filename, strlen(filename) + 1);
}
void VideoPlayer::stop() {
if (state == Stopped) {
return;
}
state = Stopped;
free();
// 通知外界
if (callback.stateChanged) {
callback.stateChanged(userData, this);
}
}
bool VideoPlayer::isPlaying() {
return state == Playing;
}
VideoPlayer::State VideoPlayer::getState() {
return this->state;
}
int VideoPlayer::getDuration() {
return fmtCtx ? fmtCtx->duration * av_q2d(AV_TIME_BASE_Q) : 0;
}
int VideoPlayer::getTime() {
return round(aTime);
}
void VideoPlayer::setTime(double seekTime) {
int duration = getDuration();
this->seekTime = round(seekTime * duration * 1.0 / 1.0);
}
void VideoPlayer::setVolumn(double volumn) {
this->volumn = round(volumn * Max);
}
int VideoPlayer::getVolumn() {
return volumn;
}
void VideoPlayer::setMute(bool mute) {
this->mute = mute;
}
bool VideoPlayer::isMute() {
return this->mute;
}
音視頻同步方案
- 采用音頻同步視頻方案
- 1.獲取當(dāng)前音頻的時(shí)間
aTime = av_q2d(aStream->time_base) * pkt.pts; - 2.獲取當(dāng)前視頻的時(shí)間
vTime = av_q2d(vStream->time_base) * pkt.dts; - 3.如果視頻包的時(shí)間大于音頻包的時(shí)間娜亿，那就需要等待對(duì)應(yīng)的音頻時(shí)鐘到達(dá)，才進(jìn)行幀渲染蚌堵，否則就原地等待
// 如果視頻包過(guò)早被解碼出來(lái)买决，那就需要等待對(duì)應(yīng)的音頻時(shí)鐘到達(dá) while (vTime > aTime && state == Playing) { SDL_Delay(5); }
- 1.獲取當(dāng)前音頻的時(shí)間
需要明確的一些時(shí)間相關(guān)的概念
- 現(xiàn)實(shí)時(shí)間
- 比如一個(gè)視頻的時(shí)長(zhǎng)是120秒，其中120秒就是現(xiàn)實(shí)時(shí)間
- 比如一個(gè)視頻播放到了第58秒吼畏，其中第58秒就是現(xiàn)實(shí)時(shí)間
- FFmpeg時(shí)間
- 時(shí)間戳（timestamp）督赤，類(lèi)型是int64_t
- 時(shí)間基（time base\unit），是時(shí)間戳的單位宫仗，類(lèi)型是AVRational
- FFmpeg時(shí)間 與 現(xiàn)實(shí)時(shí)間的轉(zhuǎn)換
- 現(xiàn)實(shí)時(shí)間 = 時(shí)間戳 * (時(shí)間基的分子 / 時(shí)間基的分母)
- 現(xiàn)實(shí)時(shí)間 = 時(shí)間戳 * av_q2d(時(shí)間基)
- 時(shí)間戳 = 現(xiàn)實(shí)時(shí)間 / (時(shí)間基的分子 / 時(shí)間基的分母)
- 時(shí)間戳 = 現(xiàn)實(shí)時(shí)間 / av_q2d(時(shí)間基)
