- 項(xiàng)目介紹
- 音頻raw數(shù)據(jù)獲取
- 音頻重采樣
- 視頻raw數(shù)據(jù)獲取
- 視頻像素轉(zhuǎn)換
- 音視頻編碼
- 音視頻pts同步
- MP4封裝
- 注意事項(xiàng)
項(xiàng)目介紹
image-20220325112028641.png
本項(xiàng)目主要實(shí)現(xiàn)功能實(shí)現(xiàn)一個(gè)屏幕錄制器袭灯，從顯卡中抓屏捉捅，從mic中獲取pcm音頻數(shù)據(jù)脱篙，
將其編碼封裝成一個(gè)MP4文件，以當(dāng)前日期存放到指定路徑中
git地址：https://gitee.com/lisiwen945/av_codec/tree/master/screen_recode
主要用到的技術(shù)：
directx3d：從顯卡中抓屏贫橙，抓取的數(shù)據(jù)是bgra格式
ffmpeg：pcm編碼成aac數(shù)據(jù)，音頻重采樣麦箍，視頻像素轉(zhuǎn)換腌紧，視頻編碼 ，音視頻封裝成mp4
qt：屏幕錄制器界面開(kāi)發(fā)夜涕，事件響應(yīng)犯犁，音頻采集
image-20220325114400254.png
音頻raw數(shù)據(jù)獲取
// 設(shè)置音頻格式
QAudioFormat fmt;
fmt.setSampleRate(m_sampleRate);
fmt.setChannelCount(m_channels);
fmt.setSampleSize(16); // 采樣精度
fmt.setSampleType(QAudioFormat::UnSignedInt);
fmt.setByteOrder(QAudioFormat::LittleEndian);
fmt.setCodec("audio/pcm");
// 創(chuàng)建音頻錄制設(shè)備（測(cè)試不插耳機(jī)會(huì)失敗，電腦自帶的錄音設(shè)備好像不行）
m_input = new QAudioInput(fmt);
// 開(kāi)始錄制
m_io = m_input->start();
LynPcmData pcmData;
int size = 1024 * 2 * 2; // 每次讀1024個(gè)樣本女器，采樣精度是16（兩個(gè)字節(jié)）酸役，雙聲道
std::unique_lock<std::mutex> lock(m_mux);
pcmData.data = (unsigned char *)malloc(size);
int readedSize = 0;
while (readedSize < size && !m_isExit)
{
int br = m_input->bytesReady();
if (br < 1024) {
usleep(100);
continue;
}
int s = 1024;
s = s < (size - readedSize) ? s : size - readedSize;
int len = m_io->read((char *)pcmData.data + readedSize, s);
readedSize += len;
}
pcmData.size = readedSize;
音頻重采樣
// 因?yàn)閍ac編碼中要求輸入的pcm數(shù)據(jù)是float類(lèi)型，而且要求是平面的（左右聲道分開(kāi)存放）驾胆，所以需要重采樣
m_asc = swr_alloc_set_opts(m_asc,
m_actx->channel_layout,m_actx->sample_fmt,m_actx->sample_rate, // 輸出格式
av_get_default_channel_layout(2), AV_SAMPLE_FMT_S16, 44100, // 輸入格式
0,0);
int ret = swr_init(m_asc);
// 構(gòu)造輸出信息
m_pcm = av_frame_alloc();
m_pcm->format = m_actx->sample_fmt;
m_pcm->channels = m_actx->channels;
m_pcm->channel_layout = m_actx->channel_layout;
m_pcm->nb_samples = pcmData->size / 4; // 樣本數(shù)量
int ret = av_frame_get_buffer(m_pcm, 0);
// 輸入數(shù)據(jù)
const uint8_t *data[AV_NUM_DATA_POINTERS] = {0};
data[0] = (uint8_t *)pcmData->data;
// 開(kāi)始轉(zhuǎn)換
int len = swr_convert(m_asc, m_pcm->data, pcmData->size / 4,
data, pcmData->size / 4);
視頻raw數(shù)據(jù)獲取
//1 創(chuàng)建directx3d對(duì)象
static IDirect3D9 *d3d = NULL;
if (!d3d)
{
d3d = Direct3DCreate9(D3D_SDK_VERSION);
}
if (!d3d) return;
//2 創(chuàng)建顯卡的設(shè)備對(duì)象
static IDirect3DDevice9 *device = NULL;
if (!device)
{
D3DPRESENT_PARAMETERS pa;
ZeroMemory(&pa, sizeof(pa));
pa.Windowed = true;
pa.Flags = D3DPRESENTFLAG_LOCKABLE_BACKBUFFER;
pa.SwapEffect = D3DSWAPEFFECT_DISCARD;
pa.hDeviceWindow = GetDesktopWindow();
d3d->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, 0,
D3DCREATE_HARDWARE_VERTEXPROCESSING, &pa, &device
);
}
if (!device) {
return;
}
//3創(chuàng)建離屏表面
int w = GetSystemMetrics(SM_CXSCREEN);
int h = GetSystemMetrics(SM_CYSCREEN);
static IDirect3DSurface9 *sur = NULL;
if (!sur)
{
device->CreateOffscreenPlainSurface(w, h,
D3DFMT_A8R8G8B8, D3DPOOL_SCRATCH, &sur, 0);
}
if (!sur)return;
//4 抓屏
device->GetFrontBufferData(0, sur);
//5 取出數(shù)據(jù)
D3DLOCKED_RECT rect;
ZeroMemory(&rect, sizeof(rect));
if (sur->LockRect(&rect, 0, 0) != S_OK)
{
return;
}
frame.w = w;
frame.h = h;
frame.dataSize = w * h * 4;
frame.data = malloc(frame.dataSize);
memcpy(frame.data , rect.pBits, frame.dataSize);
sur->UnlockRect();
視頻像素轉(zhuǎn)換
m_vsc = sws_getCachedContext(m_vsc,
frame->w, frame->h, AV_PIX_FMT_BGRA, // 輸入信息
m_outWidth, m_outHeight, AV_PIX_FMT_YUV420P, // 輸出信息
SWS_BICUBIC,
NULL,NULL,NULL);
m_yuv = av_frame_alloc();
m_yuv->format = AV_PIX_FMT_YUV420P;
m_yuv->width = m_outWidth;
m_yuv->height = m_outHeight;
m_yuv->pts = 0;
int ret = av_frame_get_buffer(m_yuv, 32);
if (ret != 0) {
std::cout << "av_frame_get_buffer failed!" << std::endl;
return false;
}
// rgb to yuv
uint8_t *indata[AV_NUM_DATA_POINTERS] = { 0 };
indata[0] = (uint8_t *)frame->data;
int insize[AV_NUM_DATA_POINTERS] = { 0 };
insize[0] = frame->w * 4;
int hight = sws_scale(m_vsc, indata, insize, 0, frame->h,
m_yuv->data, m_yuv->linesize);
CHECK_ERROR(hight <= 0, ret, "error");
音視頻編碼
// 獲取視頻解碼器
const AVCodec *videoCodec = avcodec_find_encoder(AV_CODEC_ID_H264);
// 創(chuàng)建上下文
m_vctx = avcodec_alloc_context3(videoCodec);
//比特率涣澡，壓縮后每秒大小
m_vctx->bit_rate = 4000000;
m_vctx->width = m_outWidth;
m_vctx->height = m_outHeight;
//時(shí)間基數(shù)
m_vctx->time_base = { 1, 10 };
m_vctx->framerate = { 10, 1 };
//畫(huà)面組大小，多少幀一個(gè)關(guān)鍵幀
m_vctx->gop_size = 10;
m_vctx->max_b_frames = 0;
m_vctx->pix_fmt = AV_PIX_FMT_YUV420P;
m_vctx->codec_id = AV_CODEC_ID_H264;
av_opt_set(m_vctx->priv_data, "preset", "superfast", 0);
m_vctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
// 打開(kāi)視頻編碼器
int ret = avcodec_open2(m_vctx, videoCodec, NULL);
m_videoStream = avformat_new_stream(m_ctx, NULL);
m_videoStream->codecpar->codec_tag = 0;
avcodec_parameters_from_context(m_videoStream->codecpar, m_vctx);
// 音頻編碼器
const AVCodec *audioCodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
m_actx = avcodec_alloc_context3(audioCodec);
// 設(shè)置參數(shù)
m_actx->bit_rate = 64000;
m_actx->sample_rate = 44100;
m_actx->sample_fmt = AV_SAMPLE_FMT_FLTP;
m_actx->channels = 2;
m_actx->channel_layout = av_get_default_channel_layout(2);
m_actx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
// 打開(kāi)設(shè)備
ret = avcodec_open2(m_actx, audioCodec, NULL);
m_audioStream->codecpar->codec_tag = 0;
avcodec_parameters_from_context(m_audioStream->codecpar, m_actx);
// 后面都是往解碼器灌入frame數(shù)據(jù)丧诺，獲取編碼后的包
avcodec_send_frame()
avcodec_receive_packet()
音視頻pts同步
pkt->pts = m_apts;
pkt->dts = pkt->pts;
// 音頻pts根據(jù)樣本數(shù)和采樣率和時(shí)間基數(shù)確定
m_apts += av_rescale_q(m_pcm->nb_samples, { 1, m_actx->sample_rate}, m_actx->time_base);
m_yuv->pts = m_vpts++;
// 視頻pts 根據(jù)時(shí)間基數(shù)確定入桂，時(shí)間基數(shù)和幀率相關(guān)
av_packet_rescale_ts(p, m_vctx->time_base, m_videoStream->time_base);
MP4封裝
// 封裝文件輸出上下文
avformat_alloc_output_context2(&m_ctx, NULL, NULL, fileName.c_str());
av_dump_format(m_ctx, 0, m_fileName.c_str(), 1);
// 打開(kāi)io
ret = avio_open(&m_ctx->pb, m_fileName.c_str(), AVIO_FLAG_WRITE);
// 寫(xiě)入封裝頭
ret = avformat_write_header(m_ctx, NULL);
// 將包寫(xiě)入文件
av_interleaved_write_frame(m_ctx, pkt);
// 寫(xiě)入尾部信息
av_write_trailer(m_ctx)
注意事項(xiàng)
- 音頻編碼和視頻編碼在不同的線(xiàn)程中，需要考慮多線(xiàn)程同步的問(wèn)題
- 視頻的幀率控制需要考慮屏幕抓屏和編碼的時(shí)間驳阎，不能簡(jiǎn)單的用sleep控制抗愁，得用定時(shí)器
- 要注意編解碼packet和frame的釋放時(shí)機(jī)
- 編碼器內(nèi)部會(huì)有緩存，在結(jié)束時(shí)理論上應(yīng)該將frame設(shè)置成NULL沖刷緩沖區(qū)呵晚，因?yàn)槭且曨l錄制結(jié)束缺個(gè)幾幀影響不大
