使用了很长时间的zlmediakit流媒体服务，一直对其精妙高效的设计实现十分好奇。最好的学习就是去二次开发实现一些小功能，同时摸索框架的代码结构
在参考了zlmediakit的录像功能后，分析模仿它的源码结构，实现定时抽帧的功能。
抽帧之后可以：1）进行算法分析；2）重新编码实现转码功能；3）算法分析之后再编码，实现算法结果视频流程序。
向优秀的流媒体服务zlmediakit致敬！！
--------------------------------
关键代码如下：
1.在installWebApi方法中新增开始转码和停止转码的HTTP接口
void installWebApi() { ***** api_regist("/index/api/startTranscode", [](API_ARGS_MAP_ASYNC) { CHECK_SECRET(); CHECK_ARGS("type", "vhost", "app", "stream"); auto src = MediaSource::find(allArgs["vhost"], allArgs["app"], allArgs["stream"]); if (!src) { throw ApiRetException("can not find the stream", API::NotFound); } src->getOwnerPoller()->async([=]() mutable { auto result = src->setupTranscode((Transcoder::type)allArgs["type"].as<int>(), true, allArgs["customized_path"], allArgs["max_second"].as<size_t>()); val["result"] = result; val["code"] = result ? API::Success : API::OtherFailed; val["msg"] = result ? "success" : "start record failed"; invoker(200, headerOut, val.toStyledString()); }); }); ****** api_regist("/index/api/stopTranscode", [](API_ARGS_MAP_ASYNC) { CHECK_SECRET(); CHECK_ARGS("type", "vhost", "app", "stream"); auto src = MediaSource::find(allArgs["vhost"], allArgs["app"], allArgs["stream"]); if (!src) { throw ApiRetException("can not find the stream", API::NotFound); } src->getOwnerPoller()->async([=]() mutable { auto result = src->setupTranscode( (Transcoder::type)allArgs["type"].as<int>(), false, allArgs["customized_path"], allArgs["max_second"].as<size_t>()); val["result"] = result; val["code"] = result ? API::Success : API::OtherFailed; val["msg"] = result ? "success" : "start record failed"; invoker(200, headerOut, val.toStyledString()); }); }); ***** }
2.在MediaSource、MediaSourceEvent、MediaSourceEventInterceptor、MultiMediaSourceMuxer中模仿setupRecord添加setupTranscode
调用顺序是：MediaSource::setupTranscode ---> MediaSourceEventInterceptor::setupTranscode ---> MultiMediaSourceMuxer::setupTranscode
最终执行创建转码的对象，并赋值给MultiMediaSourceMuxer::_transcode
bool MediaSource::setupTranscode(Transcoder::type type, bool start, const string &custom_path, size_t max_second) { auto listener = _listener.lock(); if (!listener) { WarnL << "未设置MediaSource的事件监听者，setupRecord失败:" << getSchema() << "/" << getVhost() << "/" << getApp() << "/" << getId(); return false; } return listener->setupTranscode(*this, type, start, custom_path, max_second); } bool MediaSourceEventInterceptor::setupTranscode(MediaSource &sender, Transcoder::type type, bool start, const string &custom_path, size_t max_second) { auto listener = _listener.lock(); if (!listener) { return false; } return listener->setupTranscode(sender, type, start, custom_path, max_second); } bool MultiMediaSourceMuxer::setupTranscode(MediaSource &sender, Transcoder::type type, bool start, const std::string &custom_path, size_t max_second) { if (start && !_transcode) { //开始转码 _transcode = makeTranscoder(sender, getTracks(), Transcoder::type_mp4, custom_path, max_second); //创建转码对象 } else if (!start && _transcode) { //停止转码 _transcode = nullptr; } return true; }
3.在zlmediakit项目中模拟Record，创建Transcode相关的对象：Transcoder、FFmpegTranscoder、FFmpegMuxer
4.Transcoder::getTranscodePath。用于获取抽帧截图的文件夹目录
std::string Transcoder::getTranscodePath( type type, const std::string &vhost, const std::string &app, const std::string &stream_id, const std::string &customized_path) { GET_CONFIG(bool, enableVhost, General::kEnableVhost); GET_CONFIG(string, recordPath, Record::kFilePath); GET_CONFIG(string, recordAppName, Record::kAppName); string mp4FilePath; if (enableVhost) { mp4FilePath = vhost + "/" + recordAppName + "/" + app + "/" + stream_id + "/"; } else { mp4FilePath = recordAppName + "/" + app + "/" + stream_id + "/"; } // Here we use the customized file path. if (!customized_path.empty()) { return File::absolutePath(mp4FilePath, customized_path); } return File::absolutePath(mp4FilePath, recordPath); }
5.Transcoder::createTranscoder。用于创建一个FFmpegTranscoder转码对象
std::shared_ptr<MediaSinkInterface> Transcoder::createTranscoder(type type, const std::string &vhost, const std::string &app, const std::string &stream_id,const std::string &customized_path, size_t max_second) { auto path = Transcoder::getTranscodePath(type, vhost, app, stream_id, customized_path); return std::make_shared<FFmpegTranscoder>(path, vhost, app, stream_id, max_second); }
6.FFmpegTranscoder::FFmpegTranscoder。继承自MediaSinkInterface，可以作为一个输出类型的MediaSink。保存info信息，并创建一个FFmpegMuxer封装器对象
FFmpegTranscoder::FFmpegTranscoder(const std::string &path, const std::string &vhost, const std::string &app, const std::string &stream_id,size_t max_second) { _folder_path = path; _info.app = app; _info.stream = stream_id; _info.vhost = vhost; _info.folder = path; GET_CONFIG(size_t, recordSec, Record::kFileSecond); _max_second = max_second ? max_second : recordSec; _muxer = std::make_shared<FFmpegMuxer>(_folder_path, _max_second); }
7.FFmpegTranscoder::addTrack。继承自MediaSink::addTrack。添加音视频轨道。最终是向封装器对象_muxer中添加轨道，暂时只关注视频轨道
bool FFmpegTranscoder::addTrack(const Track::Ptr &track) { _tracks.emplace_back(track); if (track->getTrackType() == TrackVideo) { _have_video = true; _muxer->addTrack(track); }); } return true; }
8.FFmpegTranscoder::inputFrame。继承自MediaSink::inputFrame，接收帧数据。将帧数据写入到了_muxer
bool FFmpegTranscoder::inputFrame(const Frame::Ptr &frame) { if (_muxer) { return _muxer->inputFrame(frame); } return true; }
9.FFmpegMuxer::addTrack。创建视频解码器，用的是zlmediakit已经封装好的FFmpegDecoder，是一个基于ffmpeg的多线程异步解码对象，并且可以硬件解码器。
设置解码回调:解析将解码后的YUV数据帧转成图片格式并落地保存。
bool FFmpegMuxer::addTrack(const Track::Ptr &track) { if (track->getCodecId() == CodecH264) { _video_dec.reset(new FFmpegDecoder(track)); } else if (track->getCodecId() == CodecH265) { _video_dec.reset(new FFmpegDecoder(track)); } else { } if (_video_dec != nullptr) { _video_dec->setOnDecode([this](const FFmpegFrame::Ptr &frame) { time_t now = ::time(NULL); if (now - _last_time >= _gapTime) { AVFrame *avFrame = frame->get(); int bufSize = av_image_get_buffer_size(AV_PIX_FMT_BGRA, avFrame->width, avFrame->height, 64); uint8_t *buf = (uint8_t *)av_malloc(bufSize); int picSize = frameToImage(avFrame, AV_CODEC_ID_MJPEG, buf, bufSize); if (picSize > 0) { auto file_path = _folder_path + getTimeStr("%H-%M-%S_") + std::to_string(_index) + ".jpeg"; auto f = fopen(file_path.c_str(), "wb+"); if (f) { fwrite(buf, sizeof(uint8_t), bufSize, f); fclose(f); } } av_free(buf); _index++; _last_time = now; } }); } return true; }
10.FFmpegMuxer::frameToImage。利用FFmpeg将AVFrame的视频帧转成二进制数组输出
int FFmpegMuxer::frameToImage(AVFrame *frame, AVCodecID codecID, uint8_t *outbuf, size_t outbufSize) { int ret = 0; AVPacket pkt; AVCodec *codec; AVCodecContext *ctx = NULL; AVFrame *rgbFrame = NULL; uint8_t *buffer = NULL; struct SwsContext *swsContext = NULL; av_init_packet(&pkt); codec = avcodec_find_encoder(codecID); if (!codec) { goto end; } if (!codec->pix_fmts) { goto end; } ctx = avcodec_alloc_context3(codec); ctx->bit_rate = 3000000; ctx->width = frame->width; ctx->height = frame->height; ctx->time_base.num = 1; ctx->time_base.den = 25; ctx->gop_size = 10; ctx->max_b_frames = 0; ctx->thread_count = 1; ctx->pix_fmt = *codec->pix_fmts; ret = avcodec_open2(ctx, codec, NULL); if (ret < 0) { printf("avcodec_open2 error %d", ret); goto end; } if (frame->format != ctx->pix_fmt) { rgbFrame = av_frame_alloc(); if (rgbFrame == NULL) { printf("av_frame_alloc fail"); goto end; } swsContext = sws_getContext( frame->width, frame->height, (enum AVPixelFormat)frame->format, frame->width, frame->height, ctx->pix_fmt, 1, NULL, NULL, NULL); if (!swsContext) { printf("sws_getContext fail"); goto end; } int bufferSize = av_image_get_buffer_size(ctx->pix_fmt, frame->width, frame->height, 1) * 2; buffer = (unsigned char *)av_malloc(bufferSize); if (buffer == NULL) { printf("buffer alloc fail:%d", bufferSize); goto end; } av_image_fill_arrays(rgbFrame->data, rgbFrame->linesize, buffer, ctx->pix_fmt, frame->width, frame->height, 1); if ((ret = sws_scale( swsContext, frame->data, frame->linesize, 0, frame->height, rgbFrame->data, rgbFrame->linesize)) < 0) { printf("sws_scale error %d", ret); } rgbFrame->format = ctx->pix_fmt; rgbFrame->width = ctx->width; rgbFrame->height = ctx->height; ret = avcodec_send_frame(ctx, rgbFrame); } else { ret = avcodec_send_frame(ctx, frame); } if (ret < 0) { printf("avcodec_send_frame error %d", ret); goto end; } ret = avcodec_receive_packet(ctx, &pkt); if (ret < 0) { printf("avcodec_receive_packet error %d", ret); goto end; } if (pkt.size > 0 && pkt.size <= outbufSize) memcpy(outbuf, pkt.data, pkt.size); ret = pkt.size; end: if (swsContext) { sws_freeContext(swsContext); } if (rgbFrame) { av_frame_unref(rgbFrame); av_frame_free(&rgbFrame); } if (buffer) { av_free(buffer); } av_packet_unref(&pkt); if (ctx) { avcodec_close(ctx); avcodec_free_context(&ctx); } return ret; }
11.FFmpegMuxer::inputFrame。向解码器中塞入H264或H265视频帧
bool FFmpegMuxer::inputFrame(const Frame::Ptr &frame) { if(frame->getTrackType() == TrackVideo && _video_dec != nullptr) { _video_dec->inputFrame(frame, true, false, false); if (_cb) { _cb(frame); } } return true; }
12.MultiMediaSourceMuxer::onTrackFrame。MediaSource主对象收到视频帧时会向transcode对象中写入，从而实现抽帧
bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) { GET_CONFIG(bool, modify_stamp, General::kModifyStamp); auto frame = frame_in; if (modify_stamp) { //开启了时间戳覆盖 frame = std::make_shared<FrameStamp>(frame, _stamp[frame->getTrackType()],true); } bool ret = false; if (_rtmp) { ret = _rtmp->inputFrame(frame) ? true : ret; } if (_rtsp) { ret = _rtsp->inputFrame(frame) ? true : ret; } if (_ts) { ret = _ts->inputFrame(frame) ? true : ret; } //拷贝智能指针，目的是为了防止跨线程调用设置录像相关api导致的线程竞争问题 //此处使用智能指针拷贝来确保线程安全，比互斥锁性能更优 auto hls = _hls; if (hls) { ret = hls->inputFrame(frame) ? true : ret; } auto mp4 = _mp4; if (mp4) { ret = mp4->inputFrame(frame) ? true : ret; } auto transcode = _transcode; if (transcode) { ret = transcode->inputFrame(frame) ? true : ret; } #if defined(ENABLE_MP4) if (_fmp4) { ret = _fmp4->inputFrame(frame) ? true : ret; } #endif #if defined(ENABLE_RTPPROXY) for (auto &pr : _rtp_sender) { ret = pr.second->inputFrame(frame) ? true : ret; } #endif //ENABLE_RTPPROXY return ret; }