优点：实现视频流的动态处理。
缺点：视频解码（CPU/GPU）后图像处理CPU占用率高。
在Winform中通过LibVLCSharp组件获取视频流中的每一帧图像，需要设置回调函数，主要是SetVideoFormatCallbacks和SetVideoCallbacks，其定义如下所示：
/// <summary> /// Set decoded video chroma and dimensions. /// This only works in combination with libvlc_video_set_callbacks(). /// </summary> /// <param name="formatCb">callback to select the video format (cannot be NULL)</param> /// <param name="cleanupCb">callback to release any allocated resources (or NULL)</param> public void SetVideoFormatCallbacks( MediaPlayer.LibVLCVideoFormatCb formatCb, MediaPlayer.LibVLCVideoCleanupCb? cleanupCb) { this._videoFormatCb = formatCb ?? throw new ArgumentNullException(nameof (formatCb)); this._videoCleanupCb = cleanupCb; MediaPlayer.Native.LibVLCVideoSetFormatCallbacks(this.NativeReference, MediaPlayer.VideoFormatCallbackHandle, cleanupCb == null ? (MediaPlayer.LibVLCVideoCleanupCb) null : this._videoCleanupCb); } /// <summary> /// Set callbacks and private data to render decoded video to a custom area in memory. /// Use libvlc_video_set_format() or libvlc_video_set_format_callbacks() to configure the decoded format. /// Warning /// Rendering video into custom memory buffers is considerably less efficient than rendering in a custom window as normal. /// For optimal perfomances, VLC media player renders into a custom window, and does not use this function and associated callbacks. /// It is highly recommended that other LibVLC-based application do likewise. /// To embed video in a window, use libvlc_media_player_set_xid() or equivalent depending on the operating system. /// If window embedding does not fit the application use case, then a custom LibVLC video output display plugin is required to maintain optimal video rendering performances. /// The following limitations affect performance: /// Hardware video decoding acceleration will either be disabled completely, or require(relatively slow) copy from video/DSP memory to main memory. /// Sub-pictures(subtitles, on-screen display, etc.) must be blent into the main picture by the CPU instead of the GPU. /// Depending on the video format, pixel format conversion, picture scaling, cropping and/or picture re-orientation, /// must be performed by the CPU instead of the GPU. /// Memory copying is required between LibVLC reference picture buffers and application buffers (between lock and unlock callbacks). /// </summary> /// <param name="lockCb">callback to lock video memory (must not be NULL)</param> /// <param name="unlockCb">callback to unlock video memory (or NULL if not needed)</param> /// <param name="displayCb">callback to display video (or NULL if not needed)</param> public void SetVideoCallbacks( MediaPlayer.LibVLCVideoLockCb lockCb, MediaPlayer.LibVLCVideoUnlockCb? unlockCb, MediaPlayer.LibVLCVideoDisplayCb? displayCb) { this._videoLockCb = lockCb ?? throw new ArgumentNullException(nameof (lockCb)); this._videoUnlockCb = unlockCb; this._videoDisplayCb = displayCb; MediaPlayer.Native.LibVLCVideoSetCallbacks(this.NativeReference, MediaPlayer.VideoLockCallbackHandle, unlockCb == null ? (MediaPlayer.LibVLCVideoUnlockCb) null : MediaPlayer.VideoUnlockCallbackHandle, displayCb == null ? (MediaPlayer.LibVLCVideoDisplayCb) null : MediaPlayer.VideoDisplayCallbackHandle, GCHandle.ToIntPtr(this._gcHandle)); }
1、回调方法简要说明
SetVideoFormatCallbacks方法中参数formatCb用于初始化图像数据，参数cleanupCb用于清理图像数据。SetVideoFormatCallbacks方法中参数lockCb用于对视频帧进行解码，参数unlockCb用于解锁图片缓冲区，参数displayCb用以显示图片。
2、获取视频帧数据
在Winform中，在LibVLCSharp回调函数中获取视频帧以后，需要将图像数据拷贝进图像缓存中，这可以通过内存映射文件及预设的Bitmap实现。在lockCb委托方法中，通过视频源的图像参数，可以构造出内存映射文件对象MemoryMappedFile和MemoryMappedViewAccessor，同时创建Bitmap相关对象。
var size = pitches * lines; _memoryMappedFile = MemoryMappedFile.CreateNew(null, size); _memoryMappedView = _memoryMappedFile.CreateViewAccessor(); var viewHandle = _memoryMappedView.SafeMemoryMappedViewHandle.DangerousGetHandle(); userData = viewHandle; var args = new { // ReSharper disable once RedundantAnonymousTypePropertyName width = width, // ReSharper disable once RedundantAnonymousTypePropertyName height = height, }; _synchronizationContext.Post((state) => { _bitmap = new Bitmap((int)args.width, (int)args.height, PixelFormat.Format32bppRgb); _bitmapBuffer = new byte[(int)args.width * (int)args.height * 4]; _graphics = Graphics.FromImage(_bitmap); }, null);
然后，在对视频帧解码时，将视频数据写入内存映射文件中。
Marshal.WriteIntPtr(planes, userData);
3、显示视频帧
LibVLCVideoDisplayCb类型的回调函数用于显示图像。此时内存映射文件中已经含有视频帧数据，需要将其拷贝到Bitmap中。下面代码在获取内存映射文件中的图像后，通过触发事件对外开放图像绘制接口，实现对图像的二次处理；绘制完成后触发图像渲染完毕事件，通知相关控件加载图像。
try { var bitmapData = _bitmap.LockBits( new Rectangle(0, 0, _videoWidth, _videoHeight), ImageLockMode.WriteOnly, PixelFormat.Format32bppRgb ); _memoryMappedView.ReadArray(8, _bitmapBuffer, 0, _bitmapBuffer.Length); //从内存映射视图的第九个字节开始读取图像数据 Marshal.Copy(_bitmapBuffer, 0, bitmapData.Scan0, _bitmapBuffer.Length); //将图像数据写入Bitmap _bitmap.UnlockBits(bitmapData); //帧图像已渲染完毕 FrameRendered.Invoke(_graphics); } catch (Exception e) { Console.WriteLine(e.Message); } Task.Run(() => { var image = _bitmap.Clone( new Rectangle(0, 0, _bitmap.Width, _bitmap.Height), PixelFormat.Format32bppRgb); _synchronizationContext.Post((state) => { FrameChanged.Invoke(image); //帧图像已修改 }, null); });
4、部分代码
4.1 MyVlcVideoSourceProvider类
internal class MyVlcVideoSourceProvider { public event Action<Graphics> FrameRendered = (_) => { }; public event Action<Bitmap> FrameChanged = (_) => { }; /// <summary> /// The memory mapped file that contains the picture data /// </summary> private MemoryMappedFile _memoryMappedFile; /// <summary> /// The view that contains the pointer to the buffer that contains the picture data /// </summary> private MemoryMappedViewAccessor _memoryMappedView; private int _videoWidth; private int _videoHeight; private Bitmap _bitmap; private Graphics _graphics; private byte[] _bitmapBuffer; private readonly MediaPlayer _mediaPlayer; private readonly SynchronizationContext _synchronizationContext; public MyVlcVideoSourceProvider( MediaPlayer mediaPlayer, SynchronizationContext synchronizationContext ) { _mediaPlayer = mediaPlayer; _mediaPlayer.SetVideoFormatCallbacks(VideoFormat, CleanupVideo); _mediaPlayer.SetVideoCallbacks(LockVideo, null, DisplayVideo); _synchronizationContext = synchronizationContext; } public void UnInit() { RemoveVideo(); } #region Vlc video callbacks /// <summary> /// Called by vlc when the video format is needed. This method allocats the picture buffers for vlc and tells it to set the chroma to RV32 /// </summary> /// <param name="userData">The user data that will be given to the <see cref="LockVideo"/> callback. It contains the pointer to the buffer</param> /// <param name="chroma">The chroma</param> /// <param name="width">The visible width</param> /// <param name="height">The visible height</param> /// <param name="pitches">The buffer width</param> /// <param name="lines">The buffer height</param> /// <returns>The number of buffers allocated</returns> private uint VideoFormat( // ReSharper disable RedundantAssignment ref IntPtr userData, IntPtr chroma, ref uint width, ref uint height, ref uint pitches, ref uint lines ) // ReSharper restore RedundantAssignment { FourCCConverter.ToFourCC("RV32", chroma); //Correct video width and height according to TrackInfo var media = _mediaPlayer.Media; if (media != null) { foreach (MediaTrack track in media.Tracks) { if (track.TrackType == TrackType.Video) { var trackInfo = track.Data.Video; if (trackInfo.Width > 0 && trackInfo.Height > 0) { width = trackInfo.Width; height = trackInfo.Height; if (trackInfo.SarDen != 0) { width = width * trackInfo.SarNum / trackInfo.SarDen; } } break; } } } pitches = GetAlignedDimension((width * 32) / 8, 32); lines = GetAlignedDimension(height, 32); _videoWidth = (int)width; _videoHeight = (int)height; var size = pitches * lines; _memoryMappedFile = MemoryMappedFile.CreateNew(null, size); var args = new { // ReSharper disable once RedundantAnonymousTypePropertyName width = width, // ReSharper disable once RedundantAnonymousTypePropertyName height = height, }; _synchronizationContext.Post((state) => { _bitmap = new Bitmap((int)args.width, (int)args.height, PixelFormat.Format32bppRgb); _bitmapBuffer = new byte[(int)args.width * (int)args.height * 4]; _graphics = Graphics.FromImage(_bitmap); }, null); _memoryMappedView = _memoryMappedFile.CreateViewAccessor(); var viewHandle = _memoryMappedView.SafeMemoryMappedViewHandle.DangerousGetHandle(); userData = viewHandle; return 1; } /// <summary> /// Called by Vlc when it requires a cleanup /// </summary> /// <param name="userData">The parameter is not used</param> private void CleanupVideo(ref IntPtr userData) { // This callback may be called by Dispose in the Dispatcher thread, in which case it deadlocks if we call RemoveVideo again in the same thread. _synchronizationContext.Post((state) => { RemoveVideo(); }, null); } /// <summary> /// Called by libvlc when it wants to acquire a buffer where to write /// </summary> /// <param name="userData">The pointer to the buffer (the out parameter of the <see cref="VideoFormat"/> callback)</param> /// <param name="planes">The pointer to the planes array. Since only one plane has been allocated, the array has only one value to be allocated.</param> /// <returns>The pointer that is passed to the other callbacks as a picture identifier, this is not used</returns> private IntPtr LockVideo(IntPtr userData, IntPtr planes) { Marshal.WriteIntPtr(planes, userData); return userData; } /// <summary> /// Called by libvlc when the picture has to be displayed. /// </summary> /// <param name="userData">The pointer to the buffer (the out parameter of the <see cref="VideoFormat"/> callback)</param> /// <param name="picture">The pointer returned by the <see cref="LockVideo"/> callback. This is not used.</param> private void DisplayVideo(IntPtr userData, IntPtr picture) { if (_bitmap == null) { return; } try { var bitmapData = _bitmap.LockBits( new Rectangle(0, 0, _videoWidth, _videoHeight), ImageLockMode.WriteOnly, PixelFormat.Format32bppRgb ); _memoryMappedView.ReadArray(8, _bitmapBuffer, 0, _bitmapBuffer.Length); //从内存映射视图的第九个字节开始读取图像数据 Marshal.Copy(_bitmapBuffer, 0, bitmapData.Scan0, _bitmapBuffer.Length); //将图像数据写入Bitmap _bitmap.UnlockBits(bitmapData); //帧图像已渲染完毕 FrameRendered.Invoke(_graphics); } catch (Exception e) { Console.WriteLine(e.Message); } Task.Run(() => { var image = _bitmap.Clone( new Rectangle(0, 0, _bitmap.Width, _bitmap.Height), PixelFormat.Format32bppRgb); _synchronizationContext.Post((state) => { FrameChanged.Invoke(image); //帧图像已修改 }, null); }); } #endregion /// <summary> /// Aligns dimension to the next multiple of mod /// </summary> /// <param name="dimension">The dimension to be aligned</param> /// <param name="mod">The modulus</param> /// <returns>The aligned dimension</returns> private uint GetAlignedDimension(uint dimension, uint mod) { var modResult = dimension % mod; if (modResult == 0) { return dimension; } return dimension + mod - (dimension % mod); } /// <summary> /// Removes the video (must be called from the Dispatcher thread) /// </summary> private void RemoveVideo() { _memoryMappedView?.Dispose(); _memoryMappedView = null; _memoryMappedFile?.Dispose(); _memoryMappedFile = null; _graphics?.Dispose(); _graphics = null; _bitmap?.Dispose(); _bitmap = null; } }
4.2 VideoSourceProviderControl类（基于LibVLCSharp的视频播放代码已省略）
public partial class VideoSourceProviderControl : VideoPlayerControl { public override Control VideoView => pictureBoxVideo; private readonly MyVlcVideoSourceProvider _sourceProvider; public VideoSourceProviderControl() { InitializeComponent(); _sourceProvider = new MyVlcVideoSourceProvider(MediaPlayer, SynchronizationContext.Current); _sourceProvider.FrameRendered += OnFrameRendered; _sourceProvider.FrameChanged += OnFrameChanged; } protected override void OnVideoPlayerLoad(object sender, EventArgs e) { base.OnVideoPlayerLoad(sender, e); pictureBoxVideo.SizeMode = PictureBoxSizeMode.StretchImage; pictureBoxVideo.Dock = DockStyle.Fill; } protected override void OnVideoPlayerDestroyed(object sender, EventArgs args) { base.OnVideoPlayerDestroyed(sender, args); _sourceProvider.UnInit(); } private void OnFrameRendered(Graphics graphics) { graphics.DrawString("test测试。", new Font("新宋体", 24), Brushes.Red, 100, 100); } private void OnFrameChanged(Bitmap bitmap) { var image = pictureBoxVideo.Image; pictureBoxVideo.Image = bitmap; image?.Dispose(); } }