path: root/plugins/pluginWinMF/plugin_win_mf_converter_video.cxx

/* Copyright (C) 2013-2015 Mamadou DIOP
* Copyright (C) 2013-2015 Doubango Telecom <http://www.doubango.org>
*	
* This file is part of Open Source Doubango Framework.
*
* DOUBANGO is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*	
* DOUBANGO is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*	
* You should have received a copy of the GNU General Public License
* along with DOUBANGO.
*/
/*
Video Processor MFT (http://msdn.microsoft.com/en-us/library/windows/desktop/hh162913(v=vs.85).aspx)
* The video processor supports GPU-accelerated video processing.
* The video processor MFT is a Microsoft Media Foundation transform (MFT) that performs :
	- colorspace conversion
	- video resizing
	- deinterlacing
	- frame rate conversion
	- rotation
	- cropping
	- spatial left and right view unpacking
	- and mirroring
*/
#include "plugin_win_mf_config.h"
#include "internals/mf_utils.h"

#include "tinymedia/tmedia_converter_video.h"

#include "tsk_memory.h"
#include "tsk_debug.h"

#include <assert.h>
#include <initguid.h>
#include <dmo.h>
#include <wmcodecdsp.h>

#ifdef _MSC_VER
#pragma comment(lib, "strmiids.lib")
#pragma comment(lib, "wmcodecdspuuid.lib")
#endif

EXTERN_C const GUID CLSID_VideoProcessorMFT; // defined in mf_utils.cxx

#if !defined(_WIN32_WINNT_WIN8)
#define _WIN32_WINNT_WIN8                   0x0602
#endif /* _WIN32_WINNT_WIN8 */

#if (WINVER < _WIN32_WINNT_WIN8) 
DEFINE_GUID(MF_SA_D3D11_AWARE,
	0x206b4fc8, 0xfcf9, 0x4c51, 0xaf, 0xe3, 0x97, 0x64, 0x36, 0x9e, 0x33, 0xa0);
#endif /* MF_SA_D3D11_AWARE */

#if !defined(HAVE_IMFVideoProcessorControl)
#	if defined(__IMFVideoProcessorControl_INTERFACE_DEFINED__)
#		define HAVE_IMFVideoProcessorControl 1
#	else
#		define HAVE_IMFVideoProcessorControl 0
#	endif
#endif /* HAVE_IMFVideoProcessorControl */
#if !defined(E_BOUNDS)
#	define E_BOUNDS _HRESULT_TYPEDEF_(0x8000000BL)
#endif /* E_BOUNDS */
#if !defined(PLUGIN_MF_VC_FPS)
#define PLUGIN_MF_VC_FPS		120 // Samples requires timestamp
#endif /* PLUGIN_MF_VC_FPS */

typedef struct plugin_win_mf_converter_video_ms_s
{
	TMEDIA_DECLARE_CONVERTER_VIDEO;
	
	GUID fmtSrc;
	tsk_size_t widthSrc;
	tsk_size_t heightSrc;
	
	GUID fmtDst;
	tsk_size_t widthDst;
	tsk_size_t heightDst;

	UINT32 rotation;
	UINT32 xOutputSize;
	UINT32 xInputSize;
	BOOL flip;

	IMFSample* pSampleOut;
	IMFSample* pSampleIn;

	LONGLONG rtStart;
    UINT64 rtDuration;

	IMFTransform* pMFT; // "CLSID_VideoProcessorMFT" or "CLSID_CColorConvertDMO"
#if HAVE_IMFVideoProcessorControl
	IMFVideoProcessorControl* pVPC;
#endif
	BOOL isVideoProcessor;
}
plugin_win_mf_converter_video_ms_t;

static inline const GUID& _plugin_win_mf_converter_video_ms_get_pixfmt(tmedia_chroma_t chroma);
static inline tsk_size_t _plugin_win_mf_converter_video_ms_get_size(tmedia_chroma_t chroma, tsk_size_t w, tsk_size_t h);
static inline HRESULT _plugin_win_mf_converter_video_ms_copy_rgb32_down_top(
    BYTE*       pDst,
    const BYTE* pSrc,
    INT       dwWidthInPixels,
    INT       dwHeightInPixels
    );
static HRESULT _plugin_win_mf_converter_video_ms_process_input(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample* pSample);
static HRESULT _plugin_win_mf_converter_video_ms_process_output(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample **ppSample);
static HRESULT _plugin_win_mf_converter_video_ms_process(plugin_win_mf_converter_video_ms_t* pSelf, const void* pcInputPtr, UINT32 nInputSize, IMFSample **ppSampleOut);

static int plugin_win_mf_converter_video_ms_init(tmedia_converter_video_t* self, tsk_size_t srcWidth, tsk_size_t srcHeight, tmedia_chroma_t srcChroma, tsk_size_t dstWidth, tsk_size_t dstHeight, tmedia_chroma_t dstChroma)
{
	plugin_win_mf_converter_video_ms_t* pSelf = (plugin_win_mf_converter_video_ms_t*)self;
	TSK_DEBUG_INFO("Initializing new MF Video Converter src=(%dx%d@%d) dst=(%dx%d@%d)", srcWidth, srcHeight, srcChroma, dstWidth, dstHeight, dstChroma);

	if((pSelf->fmtSrc = _plugin_win_mf_converter_video_ms_get_pixfmt(srcChroma)) == GUID_NULL)
	{
		TSK_DEBUG_ERROR("Invalid source chroma");
		return -2;
	}
	if((pSelf->fmtDst = _plugin_win_mf_converter_video_ms_get_pixfmt(dstChroma)) == GUID_NULL)
	{
		TSK_DEBUG_ERROR("Invalid destination chroma");
		return -3;
	}

	pSelf->rtStart = 0;

	pSelf->widthSrc = srcWidth;
	pSelf->heightSrc = srcHeight;
	pSelf->widthDst = dstWidth;
	pSelf->heightDst = dstHeight;
	pSelf->rotation = 0;
	pSelf->xOutputSize = (UINT32)_plugin_win_mf_converter_video_ms_get_size(dstChroma, dstWidth, dstHeight);
	pSelf->xInputSize = (UINT32)_plugin_win_mf_converter_video_ms_get_size(srcChroma, srcWidth, srcHeight);

	SafeRelease(&pSelf->pSampleOut);
	SafeRelease(&pSelf->pSampleIn);
	SafeRelease(&pSelf->pMFT);
#if HAVE_IMFVideoProcessorControl
	SafeRelease(&pSelf->pVPC);
#endif

	HRESULT hr = S_OK;

	IMFMediaType* pTypeSrc = NULL;
	IMFMediaType* pTypeDst = NULL;

	// Get video processor or Color convertor
	hr = CoCreateInstance(CLSID_VideoProcessorMFT, NULL, 
				CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&pSelf->pMFT));
	pSelf->isVideoProcessor = SUCCEEDED(hr);
	if(FAILED(hr))
	{
		TSK_DEBUG_INFO("CoCreateInstance(CLSID_VideoProcessorMFT) failed");
		if(pSelf->widthSrc == pSelf->widthDst && pSelf->heightSrc == pSelf->heightDst)
		{
			TSK_DEBUG_INFO("No video scaling is required...perform CoCreateInstance(CLSID_CColorConvertDMO)");
			CHECK_HR(hr = CoCreateInstance(CLSID_CColorConvertDMO, NULL, 
				CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&pSelf->pMFT)));
		}
		else
		{
			CHECK_HR(hr);
		}
	}



	if(pSelf->isVideoProcessor)
	{
		IMFAttributes* pAttributes = NULL;
		UINT32 GPU = 0;
		hr = pSelf->pMFT->GetAttributes(&pAttributes);
		if (SUCCEEDED(hr)) {
			hr = pAttributes->GetUINT32(MF_SA_D3D11_AWARE, &GPU);
		}
		SafeRelease(&pAttributes);
		TSK_DEBUG_INFO("MF_SA_D3D11_AWARE = %d", GPU);
#if HAVE_IMFVideoProcessorControl
		CHECK_HR(hr = pSelf->pMFT->QueryInterface(IID_PPV_ARGS(&pSelf->pVPC)));
#endif
	}


	CHECK_HR(hr = MFUtils::CreateVideoType(&pSelf->fmtSrc, &pTypeSrc, (UINT32)pSelf->widthSrc, (UINT32)pSelf->heightSrc));
	CHECK_HR(hr = MFUtils::CreateVideoType(&pSelf->fmtDst, &pTypeDst, (UINT32)pSelf->widthDst, (UINT32)pSelf->heightDst));

	CHECK_HR(hr = pSelf->pMFT->SetInputType(0, pTypeSrc, 0));
	CHECK_HR(hr = pSelf->pMFT->SetOutputType(0, pTypeDst, 0));

bail:
	SafeRelease(&pTypeSrc);
	SafeRelease(&pTypeDst);

	if(FAILED(hr))
	{
		SafeRelease(&pSelf->pMFT);
#if HAVE_IMFVideoProcessorControl
		SafeRelease(&pSelf->pVPC);
#endif
		return -4;
	}

	return 0;
}

static tsk_size_t plugin_win_mf_converter_video_ms_process(tmedia_converter_video_t* _self, const void* buffer, tsk_size_t buffer_size, void** output, tsk_size_t* output_max_size)
{
	plugin_win_mf_converter_video_ms_t* pSelf = (plugin_win_mf_converter_video_ms_t*)_self;

	HRESULT hr = S_OK;

	IMFSample *pSampleOut = NULL;
	IMFMediaBuffer* pBufferOut = NULL;

	if(!pSelf || !buffer || !output || !output_max_size)
	{
		CHECK_HR(hr = E_POINTER);
	}

	if(!pSelf->pMFT)
	{
		TSK_DEBUG_ERROR("Not initialized");
		CHECK_HR(hr = E_FAIL);
	}
#if HAVE_IMFVideoProcessorControl
	if(!pSelf->pVPC && pSelf->isVideoProcessor)
	{
		TSK_DEBUG_ERROR("Not initialized");
		CHECK_HR(hr = E_FAIL);
	}
#endif

	if(*output_max_size < pSelf->xOutputSize)
	{
		if(!(*output = tsk_realloc(*output, pSelf->xOutputSize)))
		{
			*output_max_size = 0;
			TSK_DEBUG_ERROR("Failed to allocate buffer with size = %u", pSelf->xOutputSize);
			CHECK_HR(hr = E_OUTOFMEMORY);
		}
		*output_max_size = pSelf->xOutputSize;
	}
#if HAVE_IMFVideoProcessorControl
	if(pSelf->pVPC && !!_self->flip != !!pSelf->flip)
	{
		pSelf->flip = !!_self->flip;
		CHECK_HR(hr = pSelf->pVPC->SetMirror(pSelf->flip ? MIRROR_NONE : MIRROR_VERTICAL));
	}
	if(pSelf->pVPC && _self->rotation != pSelf->rotation)
	{
		_self->rotation = pSelf->rotation;
		CHECK_HR(hr = pSelf->pVPC->SetRotation(pSelf->rotation == 0 ? ROTATION_NONE : ROTATION_NORMAL));
		
	}
#endif

	CHECK_HR(hr = _plugin_win_mf_converter_video_ms_process(
		pSelf, buffer, pSelf->xInputSize, &pSampleOut));

	if(pSampleOut) 
	{
		CHECK_HR(hr = pSampleOut->GetBufferByIndex(0, &pBufferOut));

		BYTE* pBufferPtr = NULL;
		DWORD dwDataLength = 0;
		CHECK_HR(hr = pBufferOut->GetCurrentLength(&dwDataLength));
		if(dwDataLength > 0) 
		{
			if(dwDataLength != pSelf->xOutputSize)
			{
				TSK_DEBUG_ERROR("Output size mismatch");
				CHECK_HR(hr = E_BOUNDS);
			}
			CHECK_HR(hr = pBufferOut->Lock(&pBufferPtr, NULL, NULL));

			// MFCopyImage() is optimized: MMX, SSE, or SSE2
			switch(_self->dstChroma)
			{
				// Don't waste your time guessing which parameter to use: The consumer will always request RGB32. If not used for consumer then, just memcpy()
				case tmedia_chroma_rgb32:
					{
						if(pSelf->isVideoProcessor)
						{
							hr = _plugin_win_mf_converter_video_ms_copy_rgb32_down_top(
								(BYTE*)*output, 
								(const BYTE*)pBufferPtr, 
								(INT)pSelf->widthDst, 
								(INT)pSelf->heightDst
							 );
						}
						else
						{
							hr = MFCopyImage(
								 (BYTE*)*output, 
								 (LONG)(pSelf->widthDst << 2), 
								 (BYTE*)pBufferPtr, 
								 (LONG)(pSelf->widthDst << 2), 
								 (DWORD)(pSelf->widthDst << 2), 
								 (DWORD)pSelf->heightDst
							 );
						}

						
						if(FAILED(hr))
						{
							// unlock() before leaving
							pBufferOut->Unlock();
							CHECK_HR(hr);
						}
						break;
					}
				default:
					{
						memcpy(*output, pBufferPtr, dwDataLength);
					}
			}			
			CHECK_HR(hr = pBufferOut->Unlock());
		}
	}

	pSelf->rtStart += pSelf->rtDuration;

bail:
	SafeRelease(&pSampleOut);
	SafeRelease(&pBufferOut);

	return SUCCEEDED(hr) ? pSelf->xOutputSize : 0;
}

static tsk_object_t* plugin_win_mf_converter_video_ms_ctor(tsk_object_t * self, va_list * app)
{
	plugin_win_mf_converter_video_ms_t *pSelf = (plugin_win_mf_converter_video_ms_t *)self;
	if(pSelf){
		HRESULT hr = MFFrameRateToAverageTimePerFrame(PLUGIN_MF_VC_FPS, 1, &pSelf->rtDuration);
		if(FAILED(hr)){
			pSelf->rtDuration = 83333; // 120 FPS
		}
	}
	return self;
}
static tsk_object_t* plugin_win_mf_converter_video_ms_dtor(tsk_object_t * self)
{ 
	plugin_win_mf_converter_video_ms_t *pSelf = (plugin_win_mf_converter_video_ms_t *)self;
	if(pSelf){
		SafeRelease(&pSelf->pSampleOut);
		SafeRelease(&pSelf->pSampleIn);
		SafeRelease(&pSelf->pMFT);
#if HAVE_IMFVideoProcessorControl
		SafeRelease(&pSelf->pVPC);
#endif
	}

	return self;
}
static const tsk_object_def_t plugin_win_mf_converter_video_ms_def_s = 
{
	sizeof(plugin_win_mf_converter_video_ms_t),
	plugin_win_mf_converter_video_ms_ctor, 
	plugin_win_mf_converter_video_ms_dtor,
	tsk_null, 
};
const tsk_object_def_t *plugin_win_mf_converter_video_ms_def_t = &plugin_win_mf_converter_video_ms_def_s;
static const tmedia_converter_video_plugin_def_t plugin_win_mf_converter_video_ms_plugin_def_s = 
{
	&plugin_win_mf_converter_video_ms_def_s,
	
	plugin_win_mf_converter_video_ms_init,
	plugin_win_mf_converter_video_ms_process
};
const tmedia_converter_video_plugin_def_t *plugin_win_mf_converter_video_ms_plugin_def_t = &plugin_win_mf_converter_video_ms_plugin_def_s;


static inline tsk_size_t _plugin_win_mf_converter_video_ms_get_size(tmedia_chroma_t chroma, tsk_size_t w, tsk_size_t h)
{
	switch(chroma){
		case tmedia_chroma_rgb24:
		case tmedia_chroma_bgr24:
			return (w * h * 3);
		case tmedia_chroma_rgb565le:
			return ((w * h) << 1);
		case tmedia_chroma_rgb32:
			return ((w * h) << 2);
		case tmedia_chroma_nv21:
			return ((w * h * 3) >> 1);
		case tmedia_chroma_nv12:
			return ((w * h * 3) >> 1);
		case tmedia_chroma_yuv422p:
			return ((w * h) << 1);
		case tmedia_chroma_uyvy422:
		case tmedia_chroma_yuyv422:
			return ((w * h) << 1);
		case tmedia_chroma_yuv420p:
			return ((w * h * 3) >> 1);
		default:
			TSK_DEBUG_ERROR("Invalid chroma %d", (int)chroma);
			return 0;
	}
}

static inline const GUID& _plugin_win_mf_converter_video_ms_get_pixfmt(tmedia_chroma_t chroma)
{
	switch(chroma){
		case tmedia_chroma_rgb24:
		case tmedia_chroma_bgr24:
			return MFVideoFormat_RGB24;
		case tmedia_chroma_rgb565le:
			return MFVideoFormat_RGB565;
		case tmedia_chroma_rgb32:
			return MFVideoFormat_RGB32;
		case tmedia_chroma_nv12:
			return MFVideoFormat_NV12;
		case tmedia_chroma_yuv420p:
			return MFVideoFormat_I420;
		case tmedia_chroma_yuyv422:
			return MFVideoFormat_YUY2;
		case tmedia_chroma_uyvy422:
			return MFVideoFormat_UYVY;
		default:
			TSK_DEBUG_ERROR("Invalid chroma %d", (int)chroma);
			return GUID_NULL;
	}
}

// For RGB32:
// Direct3D -> Top-Down
// Video Processor -> Down-Top
static inline HRESULT _plugin_win_mf_converter_video_ms_copy_rgb32_down_top(
    BYTE*       pDst,
    const BYTE* pSrc,
    INT       dwWidthInPixels,
    INT       dwHeightInPixels
    )
{
	RGBQUAD *pSrcPixel = &((RGBQUAD*)pSrc)[(dwWidthInPixels * dwHeightInPixels) - dwWidthInPixels];
    RGBQUAD *pDestPixel = &((RGBQUAD*)pDst)[0];

	register INT x;
	register INT y;

    for (y = dwHeightInPixels; y > 0 ; --y)
    {
        for (x = 0; x < dwWidthInPixels; ++x)
        {
            pDestPixel[x] = pSrcPixel[x];
        }
        pDestPixel += dwWidthInPixels;
        pSrcPixel -= dwWidthInPixels;
    }
	return S_OK;
}

static HRESULT _plugin_win_mf_converter_video_ms_process_input(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample* pSample)
{
	return pSelf->pMFT->ProcessInput(0, pSample, 0);
}

static HRESULT _plugin_win_mf_converter_video_ms_process_output(plugin_win_mf_converter_video_ms_t* pSelf, IMFSample **ppSample)
{
	*ppSample = NULL;

    IMFMediaBuffer* pBufferOut = NULL;

    DWORD dwStatus;

	HRESULT hr = S_OK;
  
    MFT_OUTPUT_STREAM_INFO mftStreamInfo = { 0 };
    MFT_OUTPUT_DATA_BUFFER mftOutputData = { 0 };

	CHECK_HR(hr = pSelf->pMFT->GetOutputStreamInfo(0, &mftStreamInfo));

	if(!pSelf->pSampleOut)
	{
		CHECK_HR(hr = MFUtils::CreateMediaSample(mftStreamInfo.cbSize, &pSelf->pSampleOut));
		hr = pSelf->pSampleOut->GetBufferByIndex(0, &pBufferOut);
		if(FAILED(hr))
		{
			SafeRelease(&pSelf->pSampleOut);
			CHECK_HR(hr);
		}
	}
	else
	{
		DWORD dwMaxLength = 0;
		CHECK_HR(hr = pSelf->pSampleOut->GetBufferByIndex(0, &pBufferOut));
		CHECK_HR(hr = pBufferOut->GetMaxLength(&dwMaxLength));
		if(dwMaxLength < mftStreamInfo.cbSize)
		{
			CHECK_HR(hr = pSelf->pSampleOut->RemoveAllBuffers());
			SafeRelease(&pBufferOut);
			CHECK_HR(hr = MFCreateMemoryBuffer(mftStreamInfo.cbSize, &pBufferOut));    
			CHECK_HR(hr = pSelf->pSampleOut->AddBuffer(pBufferOut));
		}
	}

	CHECK_HR(hr = pBufferOut->SetCurrentLength(0));
 
    //Set the output sample
    mftOutputData.pSample = pSelf->pSampleOut;
    //Set the output id
    mftOutputData.dwStreamID = 0;

    //Generate the output sample
    CHECK_HR(hr = pSelf->pMFT->ProcessOutput(0, 1, &mftOutputData, &dwStatus));
    /*if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT)
    {
        hr = S_OK;
        goto bail;
    }*/

    // TODO: Handle MF_E_TRANSFORM_STREAM_CHANGE

    *ppSample = pSelf->pSampleOut;
    (*ppSample)->AddRef();

bail:
    SafeRelease(&pBufferOut);
    return hr;
}

static HRESULT _plugin_win_mf_converter_video_ms_process(plugin_win_mf_converter_video_ms_t* pSelf, const void* pcInputPtr, UINT32 nInputSize, IMFSample **ppSampleOut)
{
	if(!pcInputPtr || !nInputSize || !ppSampleOut)
	{
		TSK_DEBUG_ERROR("Invalid parameter");
		return E_INVALIDARG;
	}

	*ppSampleOut = NULL;

	HRESULT hr = S_OK;
	
	IMFMediaBuffer* pBufferIn = NULL;
	BYTE* pBufferPtr = NULL;

	if(!pSelf->pSampleIn)
	{
		CHECK_HR(hr = MFUtils::CreateMediaSample(nInputSize, &pSelf->pSampleIn));
		hr = pSelf->pSampleIn->GetBufferByIndex(0, &pBufferIn);
		if(FAILED(hr))
		{
			SafeRelease(&pSelf->pSampleIn);
			CHECK_HR(hr);
		}
	}
	else
	{
		DWORD dwMaxLength = 0;
		CHECK_HR(hr = pSelf->pSampleIn->GetBufferByIndex(0, &pBufferIn));
		CHECK_HR(hr = pBufferIn->GetMaxLength(&dwMaxLength));
		if(dwMaxLength < nInputSize)
		{
			CHECK_HR(hr = pSelf->pSampleIn->RemoveAllBuffers());
			SafeRelease(&pBufferIn);
			CHECK_HR(hr = MFCreateMemoryBuffer(nInputSize, &pBufferIn));    
			CHECK_HR(hr = pSelf->pSampleIn->AddBuffer(pBufferIn));
		}
	}
	
	CHECK_HR(hr = pBufferIn->Lock(&pBufferPtr, NULL, NULL));
	memcpy(pBufferPtr, pcInputPtr, nInputSize);
	CHECK_HR(hr = pBufferIn->Unlock());
	CHECK_HR(hr = pBufferIn->SetCurrentLength(nInputSize));

	CHECK_HR(hr = pSelf->pSampleIn->SetSampleDuration(pSelf->rtDuration));
	CHECK_HR(hr = pSelf->pSampleIn->SetSampleTime(pSelf->rtStart));

	hr = _plugin_win_mf_converter_video_ms_process_input(pSelf, pSelf->pSampleIn);
	while(hr == MF_E_NOTACCEPTING)
	{
		TSK_DEBUG_INFO("MF_E_NOTACCEPTING");
		IMFSample* pSample = NULL;
		hr = _plugin_win_mf_converter_video_ms_process_output(pSelf, &pSample);
		if(SUCCEEDED(hr) && pSample)
		{
			SafeRelease(ppSampleOut);
			*ppSampleOut = pSample, pSample = NULL;

			hr = pSelf->pSampleIn->SetUINT32(MFSampleExtension_Discontinuity, TRUE);
			hr = _plugin_win_mf_converter_video_ms_process_input(pSelf, pSelf->pSampleIn);
		}		
	}
	if(!*ppSampleOut)
	{
		CHECK_HR(hr = _plugin_win_mf_converter_video_ms_process_output(pSelf, ppSampleOut));
	}
	
bail:
	SafeRelease(&pBufferIn);
	return hr;
}