/* Copyright (C) 2013-2015 Mamadou DIOP * Copyright (C) 2013-2015 Doubango Telecom * * This file is part of Open Source Doubango Framework. * * DOUBANGO is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * DOUBANGO is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with DOUBANGO. */ /* Video Processor MFT (http://msdn.microsoft.com/en-us/library/windows/desktop/hh162913(v=vs.85).aspx) * The video processor supports GPU-accelerated video processing. * The video processor MFT is a Microsoft Media Foundation transform (MFT) that performs : - colorspace conversion - video resizing - deinterlacing - frame rate conversion - rotation - cropping - spatial left and right view unpacking - and mirroring */ #include "plugin_win_mf_config.h" #include "internals/mf_utils.h" #include "tinymedia/tmedia_converter_video.h" #include "tsk_memory.h" #include "tsk_debug.h" #include #include #include #include #ifdef _MSC_VER #pragma comment(lib, "strmiids.lib") #pragma comment(lib, "wmcodecdspuuid.lib") #endif EXTERN_C const GUID CLSID_VideoProcessorMFT; // defined in mf_utils.cxx #if !defined(_WIN32_WINNT_WIN8) #define _WIN32_WINNT_WIN8 0x0602 #endif /* _WIN32_WINNT_WIN8 */ #if (WINVER < _WIN32_WINNT_WIN8) DEFINE_GUID(MF_SA_D3D11_AWARE, 0x206b4fc8, 0xfcf9, 0x4c51, 0xaf, 0xe3, 0x97, 0x64, 0x36, 0x9e, 0x33, 0xa0); #endif /* MF_SA_D3D11_AWARE */ #if !defined(HAVE_IMFVideoProcessorControl) # if defined(__IMFVideoProcessorControl_INTERFACE_DEFINED__) # define HAVE_IMFVideoProcessorControl 1 # else # define HAVE_IMFVideoProcessorControl 0 # endif #endif /* HAVE_IMFVideoProcessorControl */ #if !defined(E_BOUNDS) # define E_BOUNDS _HRESULT_TYPEDEF_(0x8000000BL) #endif /* E_BOUNDS */ #if !defined(PLUGIN_MF_VC_FPS) #define PLUGIN_MF_VC_FPS 120 // Samples requires timestamp #endif /* PLUGIN_MF_VC_FPS */ typedef struct plugin_win_mf_converter_video_ms_s { TMEDIA_DECLARE_CONVERTER_VIDEO; GUID fmtSrc; tsk_size_t widthSrc; tsk_size_t heightSrc; GUID fmtDst; tsk_size_t widthDst; tsk_size_t heightDst; UINT32 rotation; UINT32 xOutputSize; UINT32 xInputSize; BOOL flip; IMFSample* pSampleOut; IMFSample* pSampleIn; LONGLONG rtStart; UINT64 rtDuration; IMFTransform* pMFT; // "CLSID_VideoProcessorMFT" or "CLSID_CColorConvertDMO" #if HAVE_IMFVideoProcessorControl IMFVideoProcessorControl* pVPC; #endif BOOL isVideoProcessor; } plugin_win_mf_converter_video_ms_t; static inline const GUID& _plugin_win_mf_converter_video_ms_get_pixfmt(tmedia_chroma_t chroma); static inline tsk_size_t _plugin_win_mf_converter_video_ms_get_size(tmedia_chroma_t chroma, tsk_size_t w, tsk_size_t h); static inline HRESULT _plugin_win_mf_converter_video_ms_copy_rgb32_down_top( BYTE* pDst, const BYTE* pSrc, INT dwWidthInPixels, INT dwHeightInPixels ); static HRESULT _plugin_win_mf_converter_video_ms_process_input(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample* pSample); static HRESULT _plugin_win_mf_converter_video_ms_process_output(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample **ppSample); static HRESULT _plugin_win_mf_converter_video_ms_process(plugin_win_mf_converter_video_ms_t* pSelf, const void* pcInputPtr, UINT32 nInputSize, IMFSample **ppSampleOut); static int plugin_win_mf_converter_video_ms_init(tmedia_converter_video_t* self, tsk_size_t srcWidth, tsk_size_t srcHeight, tmedia_chroma_t srcChroma, tsk_size_t dstWidth, tsk_size_t dstHeight, tmedia_chroma_t dstChroma) { plugin_win_mf_converter_video_ms_t* pSelf = (plugin_win_mf_converter_video_ms_t*)self; TSK_DEBUG_INFO("Initializing new MF Video Converter src=(%dx%d@%d) dst=(%dx%d@%d)", srcWidth, srcHeight, srcChroma, dstWidth, dstHeight, dstChroma); if((pSelf->fmtSrc = _plugin_win_mf_converter_video_ms_get_pixfmt(srcChroma)) == GUID_NULL) { TSK_DEBUG_ERROR("Invalid source chroma"); return -2; } if((pSelf->fmtDst = _plugin_win_mf_converter_video_ms_get_pixfmt(dstChroma)) == GUID_NULL) { TSK_DEBUG_ERROR("Invalid destination chroma"); return -3; } pSelf->rtStart = 0; pSelf->widthSrc = srcWidth; pSelf->heightSrc = srcHeight; pSelf->widthDst = dstWidth; pSelf->heightDst = dstHeight; pSelf->rotation = 0; pSelf->xOutputSize = (UINT32)_plugin_win_mf_converter_video_ms_get_size(dstChroma, dstWidth, dstHeight); pSelf->xInputSize = (UINT32)_plugin_win_mf_converter_video_ms_get_size(srcChroma, srcWidth, srcHeight); SafeRelease(&pSelf->pSampleOut); SafeRelease(&pSelf->pSampleIn); SafeRelease(&pSelf->pMFT); #if HAVE_IMFVideoProcessorControl SafeRelease(&pSelf->pVPC); #endif HRESULT hr = S_OK; IMFMediaType* pTypeSrc = NULL; IMFMediaType* pTypeDst = NULL; // Get video processor or Color convertor hr = CoCreateInstance(CLSID_VideoProcessorMFT, NULL, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&pSelf->pMFT)); pSelf->isVideoProcessor = SUCCEEDED(hr); if(FAILED(hr)) { TSK_DEBUG_INFO("CoCreateInstance(CLSID_VideoProcessorMFT) failed"); if(pSelf->widthSrc == pSelf->widthDst && pSelf->heightSrc == pSelf->heightDst) { TSK_DEBUG_INFO("No video scaling is required...perform CoCreateInstance(CLSID_CColorConvertDMO)"); CHECK_HR(hr = CoCreateInstance(CLSID_CColorConvertDMO, NULL, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&pSelf->pMFT))); } else { CHECK_HR(hr); } } if(pSelf->isVideoProcessor) { IMFAttributes* pAttributes = NULL; UINT32 GPU = 0; hr = pSelf->pMFT->GetAttributes(&pAttributes); if (SUCCEEDED(hr)) { hr = pAttributes->GetUINT32(MF_SA_D3D11_AWARE, &GPU); } SafeRelease(&pAttributes); TSK_DEBUG_INFO("MF_SA_D3D11_AWARE = %d", GPU); #if HAVE_IMFVideoProcessorControl CHECK_HR(hr = pSelf->pMFT->QueryInterface(IID_PPV_ARGS(&pSelf->pVPC))); #endif } CHECK_HR(hr = MFUtils::CreateVideoType(&pSelf->fmtSrc, &pTypeSrc, (UINT32)pSelf->widthSrc, (UINT32)pSelf->heightSrc)); CHECK_HR(hr = MFUtils::CreateVideoType(&pSelf->fmtDst, &pTypeDst, (UINT32)pSelf->widthDst, (UINT32)pSelf->heightDst)); CHECK_HR(hr = pSelf->pMFT->SetInputType(0, pTypeSrc, 0)); CHECK_HR(hr = pSelf->pMFT->SetOutputType(0, pTypeDst, 0)); bail: SafeRelease(&pTypeSrc); SafeRelease(&pTypeDst); if(FAILED(hr)) { SafeRelease(&pSelf->pMFT); #if HAVE_IMFVideoProcessorControl SafeRelease(&pSelf->pVPC); #endif return -4; } return 0; } static tsk_size_t plugin_win_mf_converter_video_ms_process(tmedia_converter_video_t* _self, const void* buffer, tsk_size_t buffer_size, void** output, tsk_size_t* output_max_size) { plugin_win_mf_converter_video_ms_t* pSelf = (plugin_win_mf_converter_video_ms_t*)_self; HRESULT hr = S_OK; IMFSample *pSampleOut = NULL; IMFMediaBuffer* pBufferOut = NULL; if(!pSelf || !buffer || !output || !output_max_size) { CHECK_HR(hr = E_POINTER); } if(!pSelf->pMFT) { TSK_DEBUG_ERROR("Not initialized"); CHECK_HR(hr = E_FAIL); } #if HAVE_IMFVideoProcessorControl if(!pSelf->pVPC && pSelf->isVideoProcessor) { TSK_DEBUG_ERROR("Not initialized"); CHECK_HR(hr = E_FAIL); } #endif if(*output_max_size < pSelf->xOutputSize) { if(!(*output = tsk_realloc(*output, pSelf->xOutputSize))) { *output_max_size = 0; TSK_DEBUG_ERROR("Failed to allocate buffer with size = %u", pSelf->xOutputSize); CHECK_HR(hr = E_OUTOFMEMORY); } *output_max_size = pSelf->xOutputSize; } #if HAVE_IMFVideoProcessorControl if(pSelf->pVPC && !!_self->flip != !!pSelf->flip) { pSelf->flip = !!_self->flip; CHECK_HR(hr = pSelf->pVPC->SetMirror(pSelf->flip ? MIRROR_NONE : MIRROR_VERTICAL)); } if(pSelf->pVPC && _self->rotation != pSelf->rotation) { _self->rotation = pSelf->rotation; CHECK_HR(hr = pSelf->pVPC->SetRotation(pSelf->rotation == 0 ? ROTATION_NONE : ROTATION_NORMAL)); } #endif CHECK_HR(hr = _plugin_win_mf_converter_video_ms_process( pSelf, buffer, pSelf->xInputSize, &pSampleOut)); if(pSampleOut) { CHECK_HR(hr = pSampleOut->GetBufferByIndex(0, &pBufferOut)); BYTE* pBufferPtr = NULL; DWORD dwDataLength = 0; CHECK_HR(hr = pBufferOut->GetCurrentLength(&dwDataLength)); if(dwDataLength > 0) { if(dwDataLength != pSelf->xOutputSize) { TSK_DEBUG_ERROR("Output size mismatch"); CHECK_HR(hr = E_BOUNDS); } CHECK_HR(hr = pBufferOut->Lock(&pBufferPtr, NULL, NULL)); // MFCopyImage() is optimized: MMX, SSE, or SSE2 switch(_self->dstChroma) { // Don't waste your time guessing which parameter to use: The consumer will always request RGB32. If not used for consumer then, just memcpy() case tmedia_chroma_rgb32: { if(pSelf->isVideoProcessor) { hr = _plugin_win_mf_converter_video_ms_copy_rgb32_down_top( (BYTE*)*output, (const BYTE*)pBufferPtr, (INT)pSelf->widthDst, (INT)pSelf->heightDst ); } else { hr = MFCopyImage( (BYTE*)*output, (LONG)(pSelf->widthDst << 2), (BYTE*)pBufferPtr, (LONG)(pSelf->widthDst << 2), (DWORD)(pSelf->widthDst << 2), (DWORD)pSelf->heightDst ); } if(FAILED(hr)) { // unlock() before leaving pBufferOut->Unlock(); CHECK_HR(hr); } break; } default: { memcpy(*output, pBufferPtr, dwDataLength); } } CHECK_HR(hr = pBufferOut->Unlock()); } } pSelf->rtStart += pSelf->rtDuration; bail: SafeRelease(&pSampleOut); SafeRelease(&pBufferOut); return SUCCEEDED(hr) ? pSelf->xOutputSize : 0; } static tsk_object_t* plugin_win_mf_converter_video_ms_ctor(tsk_object_t * self, va_list * app) { plugin_win_mf_converter_video_ms_t *pSelf = (plugin_win_mf_converter_video_ms_t *)self; if(pSelf) { HRESULT hr = MFFrameRateToAverageTimePerFrame(PLUGIN_MF_VC_FPS, 1, &pSelf->rtDuration); if(FAILED(hr)) { pSelf->rtDuration = 83333; // 120 FPS } } return self; } static tsk_object_t* plugin_win_mf_converter_video_ms_dtor(tsk_object_t * self) { plugin_win_mf_converter_video_ms_t *pSelf = (plugin_win_mf_converter_video_ms_t *)self; if(pSelf) { SafeRelease(&pSelf->pSampleOut); SafeRelease(&pSelf->pSampleIn); SafeRelease(&pSelf->pMFT); #if HAVE_IMFVideoProcessorControl SafeRelease(&pSelf->pVPC); #endif } return self; } static const tsk_object_def_t plugin_win_mf_converter_video_ms_def_s = { sizeof(plugin_win_mf_converter_video_ms_t), plugin_win_mf_converter_video_ms_ctor, plugin_win_mf_converter_video_ms_dtor, tsk_null, }; const tsk_object_def_t *plugin_win_mf_converter_video_ms_def_t = &plugin_win_mf_converter_video_ms_def_s; static const tmedia_converter_video_plugin_def_t plugin_win_mf_converter_video_ms_plugin_def_s = { &plugin_win_mf_converter_video_ms_def_s, plugin_win_mf_converter_video_ms_init, plugin_win_mf_converter_video_ms_process }; const tmedia_converter_video_plugin_def_t *plugin_win_mf_converter_video_ms_plugin_def_t = &plugin_win_mf_converter_video_ms_plugin_def_s; static inline tsk_size_t _plugin_win_mf_converter_video_ms_get_size(tmedia_chroma_t chroma, tsk_size_t w, tsk_size_t h) { switch(chroma) { case tmedia_chroma_rgb24: case tmedia_chroma_bgr24: return (w * h * 3); case tmedia_chroma_rgb565le: return ((w * h) << 1); case tmedia_chroma_rgb32: return ((w * h) << 2); case tmedia_chroma_nv21: return ((w * h * 3) >> 1); case tmedia_chroma_nv12: return ((w * h * 3) >> 1); case tmedia_chroma_yuv422p: return ((w * h) << 1); case tmedia_chroma_uyvy422: case tmedia_chroma_yuyv422: return ((w * h) << 1); case tmedia_chroma_yuv420p: return ((w * h * 3) >> 1); default: TSK_DEBUG_ERROR("Invalid chroma %d", (int)chroma); return 0; } } static inline const GUID& _plugin_win_mf_converter_video_ms_get_pixfmt(tmedia_chroma_t chroma) { switch(chroma) { case tmedia_chroma_rgb24: case tmedia_chroma_bgr24: return MFVideoFormat_RGB24; case tmedia_chroma_rgb565le: return MFVideoFormat_RGB565; case tmedia_chroma_rgb32: return MFVideoFormat_RGB32; case tmedia_chroma_nv12: return MFVideoFormat_NV12; case tmedia_chroma_yuv420p: return MFVideoFormat_I420; case tmedia_chroma_yuyv422: return MFVideoFormat_YUY2; case tmedia_chroma_uyvy422: return MFVideoFormat_UYVY; default: TSK_DEBUG_ERROR("Invalid chroma %d", (int)chroma); return GUID_NULL; } } // For RGB32: // Direct3D -> Top-Down // Video Processor -> Down-Top static inline HRESULT _plugin_win_mf_converter_video_ms_copy_rgb32_down_top( BYTE* pDst, const BYTE* pSrc, INT dwWidthInPixels, INT dwHeightInPixels ) { RGBQUAD *pSrcPixel = &((RGBQUAD*)pSrc)[(dwWidthInPixels * dwHeightInPixels) - dwWidthInPixels]; RGBQUAD *pDestPixel = &((RGBQUAD*)pDst)[0]; register INT x; register INT y; for (y = dwHeightInPixels; y > 0 ; --y) { for (x = 0; x < dwWidthInPixels; ++x) { pDestPixel[x] = pSrcPixel[x]; } pDestPixel += dwWidthInPixels; pSrcPixel -= dwWidthInPixels; } return S_OK; } static HRESULT _plugin_win_mf_converter_video_ms_process_input(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample* pSample) { return pSelf->pMFT->ProcessInput(0, pSample, 0); } static HRESULT _plugin_win_mf_converter_video_ms_process_output(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample **ppSample) { *ppSample = NULL; IMFMediaBuffer* pBufferOut = NULL; DWORD dwStatus; HRESULT hr = S_OK; MFT_OUTPUT_STREAM_INFO mftStreamInfo = { 0 }; MFT_OUTPUT_DATA_BUFFER mftOutputData = { 0 }; CHECK_HR(hr = pSelf->pMFT->GetOutputStreamInfo(0, &mftStreamInfo)); if(!pSelf->pSampleOut) { CHECK_HR(hr = MFUtils::CreateMediaSample(mftStreamInfo.cbSize, &pSelf->pSampleOut)); hr = pSelf->pSampleOut->GetBufferByIndex(0, &pBufferOut); if(FAILED(hr)) { SafeRelease(&pSelf->pSampleOut); CHECK_HR(hr); } } else { DWORD dwMaxLength = 0; CHECK_HR(hr = pSelf->pSampleOut->GetBufferByIndex(0, &pBufferOut)); CHECK_HR(hr = pBufferOut->GetMaxLength(&dwMaxLength)); if(dwMaxLength < mftStreamInfo.cbSize) { CHECK_HR(hr = pSelf->pSampleOut->RemoveAllBuffers()); SafeRelease(&pBufferOut); CHECK_HR(hr = MFCreateMemoryBuffer(mftStreamInfo.cbSize, &pBufferOut)); CHECK_HR(hr = pSelf->pSampleOut->AddBuffer(pBufferOut)); } } CHECK_HR(hr = pBufferOut->SetCurrentLength(0)); //Set the output sample mftOutputData.pSample = pSelf->pSampleOut; //Set the output id mftOutputData.dwStreamID = 0; //Generate the output sample CHECK_HR(hr = pSelf->pMFT->ProcessOutput(0, 1, &mftOutputData, &dwStatus)); /*if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) { hr = S_OK; goto bail; }*/ // TODO: Handle MF_E_TRANSFORM_STREAM_CHANGE *ppSample = pSelf->pSampleOut; (*ppSample)->AddRef(); bail: SafeRelease(&pBufferOut); return hr; } static HRESULT _plugin_win_mf_converter_video_ms_process(plugin_win_mf_converter_video_ms_t* pSelf, const void* pcInputPtr, UINT32 nInputSize, IMFSample **ppSampleOut) { if(!pcInputPtr || !nInputSize || !ppSampleOut) { TSK_DEBUG_ERROR("Invalid parameter"); return E_INVALIDARG; } *ppSampleOut = NULL; HRESULT hr = S_OK; IMFMediaBuffer* pBufferIn = NULL; BYTE* pBufferPtr = NULL; if(!pSelf->pSampleIn) { CHECK_HR(hr = MFUtils::CreateMediaSample(nInputSize, &pSelf->pSampleIn)); hr = pSelf->pSampleIn->GetBufferByIndex(0, &pBufferIn); if(FAILED(hr)) { SafeRelease(&pSelf->pSampleIn); CHECK_HR(hr); } } else { DWORD dwMaxLength = 0; CHECK_HR(hr = pSelf->pSampleIn->GetBufferByIndex(0, &pBufferIn)); CHECK_HR(hr = pBufferIn->GetMaxLength(&dwMaxLength)); if(dwMaxLength < nInputSize) { CHECK_HR(hr = pSelf->pSampleIn->RemoveAllBuffers()); SafeRelease(&pBufferIn); CHECK_HR(hr = MFCreateMemoryBuffer(nInputSize, &pBufferIn)); CHECK_HR(hr = pSelf->pSampleIn->AddBuffer(pBufferIn)); } } CHECK_HR(hr = pBufferIn->Lock(&pBufferPtr, NULL, NULL)); memcpy(pBufferPtr, pcInputPtr, nInputSize); CHECK_HR(hr = pBufferIn->Unlock()); CHECK_HR(hr = pBufferIn->SetCurrentLength(nInputSize)); CHECK_HR(hr = pSelf->pSampleIn->SetSampleDuration(pSelf->rtDuration)); CHECK_HR(hr = pSelf->pSampleIn->SetSampleTime(pSelf->rtStart)); hr = _plugin_win_mf_converter_video_ms_process_input(pSelf, pSelf->pSampleIn); while(hr == MF_E_NOTACCEPTING) { TSK_DEBUG_INFO("MF_E_NOTACCEPTING"); IMFSample* pSample = NULL; hr = _plugin_win_mf_converter_video_ms_process_output(pSelf, &pSample); if(SUCCEEDED(hr) && pSample) { SafeRelease(ppSampleOut); *ppSampleOut = pSample, pSample = NULL; hr = pSelf->pSampleIn->SetUINT32(MFSampleExtension_Discontinuity, TRUE); hr = _plugin_win_mf_converter_video_ms_process_input(pSelf, pSelf->pSampleIn); } } if(!*ppSampleOut) { CHECK_HR(hr = _plugin_win_mf_converter_video_ms_process_output(pSelf, ppSampleOut)); } bail: SafeRelease(&pBufferIn); return hr; }