libavcodec/amfenc: Vulkan initialization support for encoder.

Added linux support for amf encoder through vulkan.

To use h.264(AMD VCE) encoder on linux amdgru-pro version 19.20+ and
amf-amdgpu-pro package(amdgru-pro contains, but does not install
automatically) are required.

This driver can be installed using amdgpu-pro-install script in
official amd driver archive.

Initialization of amf encoder occurs in this order:
1) trying to initialize through dx11(only windows)
2) trying to initialize through dx9(only windows)
3) trying to initialize through vulkan

Only Vulkan initialization available on linux.

1 files changed, 15 insertions, 2 deletions

diff --git a/libavcodec/amfenc.c b/libavcodec/amfenc.c
index 384d8ef..f66b956 100644
--- a/libavcodec/amfenc.c
+++ b/libavcodec/amfenc.c
@@ -213,6 +213,7 @@ static int amf_init_from_dxva2_device(AVCodecContext *avctx, AVDXVA2DeviceContex
 static int amf_init_context(AVCodecContext *avctx)
 {
     AmfContext *ctx = avctx->priv_data;
+    AMFContext1 *context1 = NULL;
     AMF_RESULT  res;
     av_unused int ret;
 
@@ -311,8 +312,20 @@ static int amf_init_context(AVCodecContext *avctx)
             if (res == AMF_OK) {
                 av_log(avctx, AV_LOG_VERBOSE, "AMF initialisation succeeded via D3D9.\n");
             } else {
-                av_log(avctx, AV_LOG_ERROR, "AMF initialisation failed via D3D9: error %d.\n", res);
-                return AVERROR(ENOSYS);
+                AMFGuid guid = IID_AMFContext1();
+                res = ctx->context->pVtbl->QueryInterface(ctx->context, &guid, (void**)&context1);
+                AMF_RETURN_IF_FALSE(ctx, res == AMF_OK, AVERROR_UNKNOWN, "CreateContext1() failed with error %d\n", res);
+
+                res = context1->pVtbl->InitVulkan(context1, NULL);
+                context1->pVtbl->Release(context1);
+                if (res != AMF_OK) {
+                    if (res == AMF_NOT_SUPPORTED)
+                        av_log(avctx, AV_LOG_ERROR, "AMF via Vulkan is not supported on the given device.\n");
+                    else
+                        av_log(avctx, AV_LOG_ERROR, "AMF failed to initialise on the given Vulkan device: %d.\n", res);
+                    return AVERROR(ENOSYS);
+                }
+                av_log(avctx, AV_LOG_VERBOSE, "AMF initialisation succeeded via Vulkan.\n");
             }
         }
     }