FFmpegVideoFramePool.h (6468B)
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 /* vim:set ts=2 sw=2 sts=2 et cindent: */ 3 /* This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef __FFmpegVideoFramePool_h__ 8 #define __FFmpegVideoFramePool_h__ 9 10 #include "FFmpegLibWrapper.h" 11 #include "FFmpegLibs.h" 12 #include "FFmpegLog.h" 13 #include "mozilla/layers/DMABUFSurfaceImage.h" 14 #include "mozilla/widget/DMABufDevice.h" 15 #include "mozilla/widget/DMABufSurface.h" 16 17 namespace mozilla::layers { 18 class PlanarYCbCrImage; 19 } 20 21 namespace mozilla { 22 23 // VideoFrameSurface holds a reference to GPU data with a video frame. 24 // 25 // Actual GPU pixel data are stored at DMABufSurface and 26 // DMABufSurface is passed to gecko GL rendering pipeline via. 27 // DMABUFSurfaceImage. 28 // 29 // VideoFrameSurface can optionally hold VA-API ffmpeg related data to keep 30 // GPU data locked untill we need them. 31 // 32 // VideoFrameSurface is used for both HW accelerated video decoding 33 // (VA-API) and ffmpeg SW decoding. 34 // 35 // VA-API scenario 36 // 37 // When VA-API decoding is running, ffmpeg allocates AVHWFramesContext - a pool 38 // of "hardware" frames. Every "hardware" frame (VASurface) is backed 39 // by actual piece of GPU memory which holds the decoded image data. 40 // 41 // The VASurface is wrapped by DMABufSurface and transferred to 42 // rendering queue by DMABUFSurfaceImage, where TextureClient is 43 // created and VASurface is used as a texture there. 44 // 45 // As there's a limited number of VASurfaces, ffmpeg reuses them to decode 46 // next frames ASAP even if they are still attached to DMABufSurface 47 // and used as a texture in our rendering engine. 48 // 49 // Unfortunately there isn't any obvious way how to mark particular VASurface 50 // as used. The best we can do is to hold a reference to particular AVBuffer 51 // from decoded AVFrame and AVHWFramesContext which owns the AVBuffer. 52 template <int V> 53 class VideoFrameSurface {}; 54 template <> 55 class VideoFrameSurface<LIBAV_VER>; 56 57 template <int V> 58 class VideoFramePool {}; 59 template <> 60 class VideoFramePool<LIBAV_VER>; 61 62 template <> 63 class VideoFrameSurface<LIBAV_VER> { 64 friend class VideoFramePool<LIBAV_VER>; 65 66 public: 67 NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoFrameSurface) 68 69 explicit VideoFrameSurface(DMABufSurface* aSurface, 70 VASurfaceID aFFMPEGSurfaceID); 71 72 void SetYUVColorSpace(mozilla::gfx::YUVColorSpace aColorSpace) { 73 mSurface->GetAsDMABufSurfaceYUV()->SetYUVColorSpace(aColorSpace); 74 } 75 void SetColorRange(mozilla::gfx::ColorRange aColorRange) { 76 mSurface->GetAsDMABufSurfaceYUV()->SetColorRange(aColorRange); 77 } 78 void SetColorPrimaries(mozilla::gfx::ColorSpace2 aColorPrimaries) { 79 mSurface->GetAsDMABufSurfaceYUV()->SetColorPrimaries(aColorPrimaries); 80 } 81 void SetTransferFunction(mozilla::gfx::TransferFunction aTransferFunction) { 82 mSurface->GetAsDMABufSurfaceYUV()->SetTransferFunction(aTransferFunction); 83 } 84 void SetWPChromaLocation(uint32_t aWPChromaLocation) { 85 mSurface->GetAsDMABufSurfaceYUV()->SetWPChromaLocation(aWPChromaLocation); 86 } 87 88 RefPtr<DMABufSurfaceYUV> GetDMABufSurface() { 89 return mSurface->GetAsDMABufSurfaceYUV(); 90 }; 91 92 RefPtr<layers::Image> GetAsImage(); 93 94 // Don't allow VideoFrameSurface plain copy as it leads to 95 // unexpected DMABufSurface/HW buffer releases and we don't want to 96 // deep copy them. 97 VideoFrameSurface(const VideoFrameSurface&) = delete; 98 const VideoFrameSurface& operator=(VideoFrameSurface const&) = delete; 99 100 void DisableRecycle(); 101 102 protected: 103 // Lock VAAPI related data 104 void LockVAAPIData(AVCodecContext* aAVCodecContext, AVFrame* aAVFrame, 105 const FFmpegLibWrapper* aLib); 106 // Release VAAPI related data, DMABufSurface can be reused 107 // for another frame. 108 void ReleaseVAAPIData(bool aForFrameRecycle = true); 109 110 // Check if DMABufSurface is used by any gecko rendering process 111 // (WebRender or GL compositor) or by DMABUFSurfaceImage/VideoData. 112 bool IsUsedByRenderer() const { return mSurface->IsGlobalRefSet(); } 113 114 // Surface points to dmabuf memmory owned by ffmpeg. 115 bool IsFFMPEGSurface() const { return !!mLib; } 116 117 private: 118 virtual ~VideoFrameSurface(); 119 120 const RefPtr<DMABufSurface> mSurface; 121 const FFmpegLibWrapper* mLib; 122 AVBufferRef* mAVHWFrameContext; 123 AVBufferRef* mHWAVBuffer; 124 VASurfaceID mFFMPEGSurfaceID; 125 bool mHoldByFFmpeg; 126 }; 127 128 // VideoFramePool class is thread-safe. 129 template <> 130 class VideoFramePool<LIBAV_VER> { 131 public: 132 explicit VideoFramePool(int aFFMPEGPoolSize); 133 ~VideoFramePool(); 134 135 RefPtr<VideoFrameSurface<LIBAV_VER>> GetVideoFrameSurface( 136 VADRMPRIMESurfaceDescriptor& aVaDesc, int aWidth, int aHeight, 137 AVCodecContext* aAVCodecContext, AVFrame* aAVFrame, 138 const FFmpegLibWrapper* aLib); 139 RefPtr<VideoFrameSurface<LIBAV_VER>> GetVideoFrameSurface( 140 AVDRMFrameDescriptor& aDesc, int aWidth, int aHeight, 141 AVCodecContext* aAVCodecContext, AVFrame* aAVFrame, 142 const FFmpegLibWrapper* aLib); 143 RefPtr<VideoFrameSurface<LIBAV_VER>> GetVideoFrameSurface( 144 const layers::PlanarYCbCrData& aData, AVCodecContext* aAVCodecContext); 145 146 void ReleaseUnusedVAAPIFrames(); 147 void FlushFFmpegFrames(); 148 149 private: 150 RefPtr<VideoFrameSurface<LIBAV_VER>> GetTargetVideoFrameSurfaceLocked( 151 const MutexAutoLock& aProofOfLock, VASurfaceID aFFmpegSurfaceID, 152 bool aRecycleSurface); 153 RefPtr<VideoFrameSurface<LIBAV_VER>> GetFFmpegVideoFrameSurfaceLocked( 154 const MutexAutoLock& aProofOfLock, VASurfaceID aFFMPEGSurfaceID); 155 RefPtr<VideoFrameSurface<LIBAV_VER>> GetFreeVideoFrameSurfaceLocked( 156 const MutexAutoLock& aProofOfLock); 157 bool ShouldCopySurface(); 158 159 private: 160 // Protect mDMABufSurfaces pool access 161 Mutex mSurfaceLock MOZ_UNANNOTATED; 162 nsTArray<RefPtr<VideoFrameSurface<LIBAV_VER>>> mDMABufSurfaces; 163 // Maximal number of dmabuf surfaces allocated by ffmpeg for decoded video 164 // frames. Can be adjusted by extra_hw_frames at InitVAAPICodecContext(). 165 // Zero meand unlimited / dynamically allocated pool. 166 int mMaxFFMPEGPoolSize; 167 // We may fail to create texture over DMABuf memory due to driver bugs so 168 // check that before we export first DMABuf video frame. 169 Maybe<bool> mTextureCreationWorks; 170 // We may fail to copy DMABuf memory on NVIDIA drivers. 171 bool mTextureCopyWorks = true; 172 }; 173 174 } // namespace mozilla 175 176 #endif // __FFmpegVideoFramePool_h__