今天我們來談?wù)凙ndroid 環(huán)境下幅聘，使用FFmpeg錄制視頻的流程。
基本流程解析
使用FFmpeg錄制視頻的流程大體如下：
1余寥、初始化FFmpeg
2领铐、打開音頻流、視頻流
3宋舷、將PCM編碼為AAC
4绪撵、將YUV編碼為H264
5、寫入文件
6祝蝠、寫入文件尾部信息
7音诈、關(guān)閉媒體流
初始化FFmpeg
初始化FFmpeg，主要是有一下幾個步驟：
1绎狭、注冊所有的編碼器
2改艇、分配輸出格式上下文(AVFormatContext)
3、打開視頻流坟岔，并初始化視頻編碼器
4、打開音頻流摔桦，并初始化音頻編碼器
5摔握、打開視頻編碼器
6蠕趁、打開音頻編碼器
7、寫入文件頭不信息
整個過程實(shí)現(xiàn)代碼如下：
/**
* 初始化編碼器
* @return
*/
bool CainEncoder::initEncoder() {
if (isInited) {
return true;
}
do {
AVDictionary *opt = NULL;
int ret;
av_log_set_callback(ffmpeg_log);
// 注冊
av_register_all();
// 分配輸出媒體上下文
avformat_alloc_output_context2(&fmt_ctx, NULL, NULL, mOutputFile);
if (fmt_ctx == NULL) {
ALOGE("fail to avformat_alloc_output_context2 for %s", mOutputFile);
break;
}
// 獲取AVOutputFormat
fmt = fmt_ctx->oformat;
// 使用默認(rèn)格式編碼器視頻流，并初始化編碼器
if (fmt->video_codec != AV_CODEC_ID_NONE) {
addStream(&video_st, fmt_ctx, &video_codec, fmt->video_codec);
have_video = true;
}
// 使用默認(rèn)格式編碼器音頻流碘箍，并初始化編碼器
if (fmt->audio_codec != AV_CODEC_ID_NONE && enableAudio) {
addStream(&audio_st, fmt_ctx, &audio_codec, fmt->audio_codec);
have_audio = true;
}
if(!have_video && !have_audio) {
ALOGE("no audio or video codec found for the fmt!");
break;
}
// 打開視頻編碼器
if (have_video) {
openVideo(video_codec, &video_st, opt);
}
// 打開音頻編碼器
if (have_audio) {
openAudio(audio_codec, &audio_st, opt);
}
// 打開輸出文件
ret = avio_open(&fmt_ctx->pb, mOutputFile, AVIO_FLAG_READ_WRITE);
if (ret < 0) {
ALOGE("Could not open '%s': %s", mOutputFile, av_err2str(ret));
break;
}
// 寫入文件頭部信息
ret = avformat_write_header(fmt_ctx, NULL);
if (ret < 0) {
ALOGE("Error occurred when opening output file: %s", av_err2str(ret));
break;
}
isInited = true;
} while (0);
// 判斷是否初始化成功，如果不成功甜孤，則需要重置所有狀態(tài)
if (!isInited) {
reset();
}
return isInited;
}
其中邻薯，打開媒體流的方法如下：
/**
* 添加碼流
* @param ost
* @param oc
* @param codec
* @param codec_id
* @return
*/
bool CainEncoder::addStream(OutputStream *ost, AVFormatContext *oc, AVCodec **codec,
enum AVCodecID codec_id) {
AVCodecContext *context;
// 查找編碼器
*codec = avcodec_find_encoder(codec_id);
if (!(*codec)) {
ALOGE("Could not find encoder for '%s'\n", avcodec_get_name(codec_id));
return false;
}
// 創(chuàng)建輸出碼流
ost->st = avformat_new_stream(oc, *codec);
if (!ost->st) {
ALOGE("Could not allocate stream\n");
return false;
}
// 綁定碼流的ID
ost->st->id = oc->nb_streams - 1;
// 創(chuàng)建編碼上下文
context = avcodec_alloc_context3(*codec);
if (!context) {
ALOGE("Could not alloc an encoding context\n");
return false;
}
// 綁定編碼上下文
ost->enc = context;
// 判斷編碼器的類型
switch ((*codec)->type) {
// 如果創(chuàng)建的是音頻碼流,則設(shè)置音頻編碼器的參數(shù)
case AVMEDIA_TYPE_AUDIO:
context->sample_fmt = (*codec)->sample_fmts
? (AVSampleFormat) (*codec)->sample_fmts[0]
: AV_SAMPLE_FMT_S16;
context->bit_rate = mAudioBitRate;
context->sample_rate = mAudioSampleRate;
// 判斷支持的采樣率
if ((*codec)->supported_samplerates) {
context->sample_rate = (*codec)->supported_samplerates[0];
for (int i = 0; (*codec)->supported_samplerates[i]; i++) {
if((*codec)->supported_samplerates[i] == mAudioSampleRate) {
context->sample_rate = mAudioSampleRate;
}
}
}
// 設(shè)定聲道
context->channels = av_get_channel_layout_nb_channels(context->channel_layout);
context->channel_layout = AV_CH_LAYOUT_STEREO;
if ((*codec)->channel_layouts) {
context->channel_layout = (*codec)->channel_layouts[0];
for (int i = 0; (*codec)->channel_layouts[i]; i++) {
if ((*codec)->channel_layouts[i] == AV_CH_LAYOUT_STEREO) {
context->channel_layout = AV_CH_LAYOUT_STEREO;
}
}
}
// 重新設(shè)定聲道
context->channels = av_get_channel_layout_nb_channels(context->channel_layout);
// 設(shè)定time_base
ost->st->time_base = (AVRational) {1, context->sample_rate};
break;
// 如果創(chuàng)建的是視頻碼流，則設(shè)置視頻編碼器的參數(shù)
case AVMEDIA_TYPE_VIDEO:
context->codec_id = codec_id;
context->bit_rate = mBitRate;
context->width = mWidth;
context->height = mHeight;
ost->st->time_base = (AVRational) {1, mFrameRate};
context->time_base = ost->st->time_base;
context->gop_size = 12;
context->pix_fmt = AV_PIX_FMT_YUV420P;
context->thread_count = 12;
context->qmin = 10;
context->qmax = 51;
context->max_b_frames = 3;
break;
default:
break;
}
// 全局頭部信息是否存在
if (oc->oformat->flags & AVFMT_GLOBALHEADER) {
context->flags |= CODEC_FLAG_GLOBAL_HEADER;
}
return true;
}
打開音頻編碼器方法如下：
/**
* 打開音頻編碼器
* @param codec
* @param ost
* @param opt_arg
* @return
*/
bool CainEncoder::openAudio(AVCodec *codec, OutputStream *ost, AVDictionary *opt_arg) {
AVCodecContext * codecContext;
int ret;
AVDictionary *opt = NULL;
// 獲取音頻編碼上下文
codecContext = ost->enc;
// 復(fù)制信息
av_dict_copy(&opt, opt_arg, 0);
// 打開音頻編碼器
ret = avcodec_open2(codecContext, codec, &opt);
av_dict_free(&opt);
if (ret < 0) {
ALOGE("Could not open audio codec: %s", av_err2str(ret));
return false;
}
// 創(chuàng)建音頻編碼的AVFrame
ost->frame = allocAudioFrame(codecContext->channels, codecContext->sample_fmt,
codecContext->channel_layout,
codecContext->sample_rate, codecContext->frame_size);
// 創(chuàng)建暫存的AVFrame
ost->tmp_frame = allocAudioFrame(codecContext->channels, AV_SAMPLE_FMT_S16,
codecContext->channel_layout,
codecContext->sample_rate, codecContext->frame_size);
// 將碼流參數(shù)復(fù)制到復(fù)用器
ret = avcodec_parameters_from_context(ost->st->codecpar, codecContext);
if (ret < 0) {
ALOGE("Could not copy the stream parameters\n");
return false;
}
// 創(chuàng)建重采樣上下文
ost->swr_ctx = swr_alloc();
if (!ost->swr_ctx) {
ALOGE("Could not allocate resampler context");
return false;
}
// 設(shè)定重采樣信息
av_opt_set_int(ost->swr_ctx, "in_channel_count", codecContext->channels, 0);
av_opt_set_int(ost->swr_ctx, "in_sample_rate", codecContext->sample_rate, 0);
av_opt_set_sample_fmt(ost->swr_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_int(ost->swr_ctx, "out_channel_count", codecContext->channels, 0);
av_opt_set_int(ost->swr_ctx, "out_sample_rate", codecContext->sample_rate, 0);
av_opt_set_sample_fmt(ost->swr_ctx, "out_sample_fmt", codecContext->sample_fmt, 0);
// 初始化音頻重采樣上下文
ret = swr_init(ost->swr_ctx);
if (ret < 0) {
ALOGE("Failed to initialize the resampling context");
return false;
}
return true;
}
創(chuàng)建音頻幀方法如下：
/**
* 創(chuàng)建音頻編碼幀
* @param sample_fmt
* @param channel_layout
* @param sample_rate
* @param frame_size
* @return
*/
AVFrame* CainEncoder::allocAudioFrame(int channels, enum AVSampleFormat sample_fmt,
uint64_t channel_layout, int sample_rate, int frame_size) {
// 創(chuàng)建音頻幀
AVFrame *frame = av_frame_alloc();
int ret;
if (!frame) {
ALOGE("Error allocating an audio frame");
return NULL;
}
// 設(shè)定音頻幀的格式
frame->format = sample_fmt;
frame->channel_layout = channel_layout;
frame->sample_rate = sample_rate;
frame->nb_samples = frame_size;
// 設(shè)置采樣的緩沖大小
audioSampleSize = av_samples_get_buffer_size(NULL, channels, frame_size,
sample_fmt, 1);
// 創(chuàng)建緩沖區(qū)
uint8_t *frame_buf = (uint8_t *) av_malloc(audioSampleSize);
// 填充音頻緩沖數(shù)據(jù)
avcodec_fill_audio_frame(frame, channels, sample_fmt,
(const uint8_t *) frame_buf, audioSampleSize, 1);
return frame;
}
打開視頻編碼器如下：
/**
* 打開視頻編碼器
* @param codec
* @param ost
* @param opt_arg
* @return
*/
bool CainEncoder::openVideo(AVCodec *codec, OutputStream *ost, AVDictionary *opt_arg) {
int ret;
// 獲取視頻編碼上下文
AVCodecContext *codecContext = ost->enc;
AVDictionary *opt = NULL;
av_dict_copy(&opt, opt_arg, 0);
// 設(shè)定H264編碼參數(shù)御滩，如果不設(shè)定鸥拧，編碼會有延遲
if (ost->enc->codec_id == AV_CODEC_ID_H264) {
av_dict_set(&opt, "tune", "zerolatency", 0);
av_opt_set(codecContext->priv_data, "preset", "ultrafast", 0);
av_dict_set(&opt, "profile", "baseline", 0);
}
// 打開視頻編碼器
ret = avcodec_open2(codecContext, codec, &opt);
av_dict_free(&opt);
if (ret < 0) {
ALOGE("Could not open video codec: %s", av_err2str(ret));
return false;
}
// 分配并初始化一個可重用的幀
ost->frame = allocVideoFrame(codecContext->pix_fmt,
codecContext->width, codecContext->height);
if (!ost->frame) {
ALOGE("Could not allocate video frame");
return false;
}
// 如果輸出格式不是YUV420P党远，那么也需要臨時的YUV420P圖像。 然后將其轉(zhuǎn)換為所需的輸出格式
ost->tmp_frame = NULL;
if (codecContext->pix_fmt != mPixFmt) {
ost->tmp_frame = allocVideoFrame(mPixFmt, codecContext->width, codecContext->height);
if (!ost->tmp_frame) {
ALOGE("Could not allocate temporary picture");
return false;
}
}
// 將碼流參數(shù)復(fù)制到復(fù)用器
ret = avcodec_parameters_from_context(ost->st->codecpar, codecContext);
if (ret < 0) {
ALOGE("Could not copy the stream parameters\n");
return false;
}
return true;
}
創(chuàng)建視頻幀方法如下：
/**
* 創(chuàng)建視頻幀
* @param pix_fmt
* @param width
* @param height
* @return
*/
AVFrame* CainEncoder::allocVideoFrame(enum AVPixelFormat pix_fmt, int width, int height) {
AVFrame *picture;
int ret;
// 創(chuàng)建AVFrame
picture = av_frame_alloc();
if (!picture) {
return NULL;
}
picture->format = pix_fmt;
picture->width = width;
picture->height = height;
// 創(chuàng)建緩沖區(qū)
int picture_size = avpicture_get_size(pix_fmt, width, height);
uint8_t *buf = (uint8_t *) av_malloc(picture_size);
avpicture_fill((AVPicture *) picture, buf, pix_fmt, width, height);
return picture;
}
到這里富弦，我們就基本把錄制視頻需要的FFmpeg環(huán)境基本搭好沟娱，并且成功打開音頻流和視頻流。接下來腕柜，我們需要編寫將PCM編碼為AAC 以及將YUV編碼為H264 的方法济似。
音頻編碼、視頻編碼
視頻編碼盏缤。視頻編碼通常是將YUV編碼為H264砰蠢。基本編碼流程如下：
1唉铜、獲取視頻編碼上下文
2台舱、根據(jù)YUV的格式復(fù)制數(shù)據(jù)
3、根據(jù)格式判斷是否需要進(jìn)行轉(zhuǎn)碼
4打毛、對視頻進(jìn)行編碼柿赊，將YUV格式編碼為H264
5、寫入文件
實(shí)現(xiàn)方法如下：
/**
* 視頻編碼
* @param data
* @return
*/
status_t CainEncoder::videoEncode(uint8_t *data) {
int ret;
// 獲取輸出碼流
OutputStream *ost = &video_st;
AVCodecContext *context;
AVFrame *frame;
int got_frame = 0;
// 獲取視頻編碼上下文
context = ost->enc;
// 根據(jù)格式復(fù)制數(shù)據(jù)
if (mPixFmt == AV_PIX_FMT_NV21) {
memcpy(ost->tmp_frame->data[0], data, context->width * context->height);
memcpy(ost->tmp_frame->data[1], (char *) data + context->width * context->height,
context->width * context->height / 2);
} else if (mPixFmt == AV_PIX_FMT_YUV420P) { // YUV420P格式復(fù)制
memcpy(ost->frame->data[0], data, context->width * context->height);
memcpy(ost->frame->data[1], (char *) data + context->width * context->height,
context->width * context->height / 4);
memcpy(ost->frame->data[2], (char *) data + context->width * context->height * 5 / 4,
context->width * context->height / 4);
}
// 判斷格式是否相同幻枉，不相同時碰声，必須進(jìn)行轉(zhuǎn)換
if (context->pix_fmt != mPixFmt) {
if (!ost->sws_ctx) {
ost->sws_ctx = sws_getContext(context->width, context->height,
mPixFmt,
context->width, context->height,
context->pix_fmt,
SWS_BICUBIC, NULL, NULL, NULL);
if (!ost->sws_ctx) {
ALOGE("Could not initialize the conversion context");
return UNKNOWN_ERROR;
}
}
// 格式轉(zhuǎn)換
sws_scale(ost->sws_ctx,
(const uint8_t *const *) ost->tmp_frame->data, ost->tmp_frame->linesize,
0, context->height, ost->frame->data, ost->frame->linesize);
}
// 計(jì)算AVFrame的pts
ost->frame->pts = av_rescale_q(ost->next_pts++,
(AVRational) {1, mFrameRate}, ost->st->time_base);
frame = ost->frame;
// 初始化一個AVPacket
AVPacket pkt = {0};
av_init_packet(&pkt);
// 對視頻幀進(jìn)行編碼
ret = avcodec_encode_video2(context, &pkt, frame, &got_frame);
if (ret < 0) {
ALOGE("Error encoding video frame: %s", av_err2str(ret));
return UNKNOWN_ERROR;
}
ALOGI("encode video frame sucess! got frame = %d\n", got_frame);
// 編碼成功則將數(shù)據(jù)寫入文件
if (got_frame == 1) {
ret = writeFrame(fmt_ctx, &context->time_base, ost->st, &pkt);
// 釋放AVPacket
av_free_packet(&pkt);
if (ret < 0) {
ALOGE("Error write video frame: %s", av_err2str(ret));
return UNKNOWN_ERROR;
}
ALOGI("write video frame sucess!\n");
} else {
ret = 0;
// 釋放AVPacket
av_free_packet(&pkt);
}
// 判斷是否寫入成功
if (ret < 0) {
ALOGE("Error while writing video frame: %s", av_err2str(ret));
return UNKNOWN_ERROR;
}
return OK;
}
音頻編碼。音頻編碼通常需要將PCM編碼為AAC熬甫∫忍簦基本的音頻編碼流程如下：
1、獲取音頻編碼上下文
2椿肩、復(fù)制需要編碼的數(shù)據(jù)
3瞻颂、根據(jù)設(shè)置判斷是否需要對PCM進(jìn)行音頻重采樣
4、將PCM編碼為AAC
5郑象、寫入文件
實(shí)現(xiàn)方法如下：
/**
* 音頻編碼
* @param data
* @param len
* @return
*/
status_t CainEncoder::audioEncode(uint8_t *data, int len) {
AVCodecContext *context;
AVFrame *frame = NULL;
int ret;
int got_frame;
int dst_nb_samples;
OutputStream *ost = &audio_st;
// 獲取源數(shù)據(jù)
unsigned char *srcData = (unsigned char *) data;
// 初始化AVPacket
AVPacket pkt = {0};
av_init_packet(&pkt);
// 獲取音頻編碼上下文
context = ost->enc;
// 獲取暫存的編碼幀
frame = audio_st.tmp_frame;
// 復(fù)制數(shù)據(jù)
memcpy(frame->data[0], srcData, len);
// 獲取pts
frame->pts = audio_st.next_pts;
// 計(jì)算pts
audio_st.next_pts += frame->nb_samples;
ALOGI("nb_samples = %d", frame->nb_samples);
// 如果音頻編碼幀存在贡这，則進(jìn)入音頻編碼階段
if (frame) {
// TODO 計(jì)算輸出的dst_nb_samples，否則沒法輸出聲音
// 這是因?yàn)樾掳姹镜腇Fmpeg音頻編碼格式已經(jīng)變成了AV_SAMPLE_FMT_FLTP
// 但輸入的PCM數(shù)據(jù)依舊是AV_SAMPLE_FMT_S16
// 轉(zhuǎn)換為目標(biāo)格式
ret = swr_convert(ost->swr_ctx, ost->frame->data, dst_nb_samples,
(const uint8_t **) frame->data, frame->nb_samples);
if (ret < 0) {
ALOGE("Error while converting\n");
return UNKNOWN_ERROR;
}
// 獲得音頻幀并設(shè)置pts等
frame = ost->frame;
frame->pts = av_rescale_q(ost->samples_count, (AVRational) {1, context->sample_rate},
context->time_base);
ost->samples_count += dst_nb_samples;
ALOGI("dst_nb_samples = %d", dst_nb_samples);
}
// 音頻編碼
ret = avcodec_encode_audio2(context, &pkt, frame, &got_frame);
if (ret < 0) {
ALOGE("Error encoding audio frame: %s\n", av_err2str(ret));
return UNKNOWN_ERROR;
}
ALOGI("encode audio frame sucess! got frame = %d\n", got_frame);
pkt.pts = frame->pts;
// 如果編碼成功厂榛，則寫入文件
if (got_frame) {
ret = writeFrame(fmt_ctx, &context->time_base, ost->st, &pkt);
// 釋放資源
av_free_packet(&pkt);
if (ret < 0) {
ALOGE("Error while writing audio frame: %s\n", av_err2str(ret));
return UNKNOWN_ERROR;
}
ALOGI("writing audio frame sucess!\n");
}
// 釋放資源
av_free_packet(&pkt);
return OK;
}
寫入文件尾部信息
在錄制完成后盖矫，需要對錄制文件寫入文件尾部信息，否則會出現(xiàn)錄制完無法播放的情況击奶，因?yàn)殇浿频玫降奈募煌暾菜懭胛募膊啃畔⑷缦拢?/p>
/**
* 停止編碼
* @return
*/
status_t CainEncoder::stopEncode() {
// 寫入文件尾
av_write_trailer(fmt_ctx);
ALOGI("寫入文件尾部");
return OK;
}
關(guān)閉媒體流
錄制完成后，我們需要關(guān)閉媒體流以及銷毀編碼上下文柜砾、編碼器等對象湃望，方式內(nèi)存泄漏。實(shí)現(xiàn)方法如下：
/**
* 關(guān)閉編碼器碼流
* @param oc
* @param ost
*/
void CainEncoder::closeStream(AVFormatContext *oc, OutputStream *ost) {
// 關(guān)閉編碼上下文
if (ost->enc != NULL) {
avcodec_free_context(&ost->enc);
}
// 釋放AVFrame
if (ost->frame != NULL) {
av_frame_free(&ost->frame);
ost->frame = NULL;
}
if (ost->tmp_frame != NULL) {
av_frame_free(&ost->tmp_frame);
ost->tmp_frame = NULL;
}
// 釋放視頻格式縮放轉(zhuǎn)換上下文
if (ost->sws_ctx != NULL) {
sws_freeContext(ost->sws_ctx);
ost->sws_ctx = NULL;
}
// 釋放重采樣上下文
if (ost->swr_ctx != NULL) {
swr_free(&ost->swr_ctx);
ost->swr_ctx = NULL;
}
}
至此，F(xiàn)Fmpeg的錄制編碼核心功能就完成了证芭。接下來就是通過調(diào)用Camera攝像頭錄制視頻和錄音了瞳浦。這里就不在詳細(xì)介紹了。詳細(xì)的錄制實(shí)現(xiàn)流程檩帐，可以參考本人的項(xiàng)目：
CainCamera
FFmpeg錄制代碼在ffmpeglibrary module 下的encoder目錄下术幔。錄制部分目前還沒有加入隊(duì)列，有興趣的同學(xué)可以在這基礎(chǔ)上添加隊(duì)列湃密，將需要錄制的YUV 和 PCM 存放到隊(duì)列中诅挑，并添加錄制同步代碼。
