From 46831b421d7ac477b32a922c8ffe3ca5acc3ba4b Mon Sep 17 00:00:00 2001
From: Dom Cobley <popcornmix@gmail.com>
Date: Mon, 6 Feb 2023 15:19:51 +0000
Subject: [PATCH 04/13] DVDVideoCodecDRMPRIME: Add support for arbitrary output
 pixel formats

This enables any ffmpeg pixel formats to be supported by DRMPRIME decoder
by creating a scale ffmpeg filter to convert it to a supported format.

This allows formats like h264 Hi10P and hevc 12-bit 444 to be software decoded,
converted and displayed through DRM.

This will be a cheaper path than disabling DRMPRIME, which is also
software decode, convert, but then needs convert to texture and display through GL.

And it happens automatically without requiring user video settings
---
 .../DVDCodecs/Video/DVDVideoCodecDRMPRIME.cpp | 124 +++++++++++-------
 .../DVDCodecs/Video/DVDVideoCodecDRMPRIME.h   |   3 +-
 2 files changed, 77 insertions(+), 50 deletions(-)

diff --git a/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.cpp b/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.cpp
index 89ad921dbe3a..94c26aa29e4b 100644
--- a/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.cpp
+++ b/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.cpp
@@ -215,7 +215,7 @@ enum AVPixelFormat CDVDVideoCodecDRMPRIME::GetFormat(struct AVCodecContext* avct
 {
   for (int n = 0; fmt[n] != AV_PIX_FMT_NONE; n++)
   {
-    if (IsSupportedHwFormat(fmt[n]) || IsSupportedSwFormat(fmt[n]))
+    //if (IsSupportedHwFormat(fmt[n]) || IsSupportedSwFormat(fmt[n]))
     {
       CDVDVideoCodecDRMPRIME* ctx = static_cast<CDVDVideoCodecDRMPRIME*>(avctx->opaque);
       ctx->UpdateProcessInfo(avctx, fmt[n]);
@@ -236,7 +236,8 @@ enum AVPixelFormat CDVDVideoCodecDRMPRIME::GetFormat(struct AVCodecContext* avct
 
 int CDVDVideoCodecDRMPRIME::GetBuffer(struct AVCodecContext* avctx, AVFrame* frame, int flags)
 {
-  if (IsSupportedSwFormat(static_cast<AVPixelFormat>(frame->format)))
+  AVPixelFormat pix_fmt = static_cast<AVPixelFormat>(frame->format);
+  if (IsSupportedSwFormat(pix_fmt))
   {
     int width = frame->width;
     int height = frame->height;
@@ -244,7 +245,7 @@ int CDVDVideoCodecDRMPRIME::GetBuffer(struct AVCodecContext* avctx, AVFrame* fra
     AlignedSize(avctx, width, height);
 
     int size;
-    switch (avctx->pix_fmt)
+    switch (pix_fmt)
     {
       case AV_PIX_FMT_YUV420P:
       case AV_PIX_FMT_YUVJ420P:
@@ -264,13 +265,12 @@ int CDVDVideoCodecDRMPRIME::GetBuffer(struct AVCodecContext* avctx, AVFrame* fra
 
     CDVDVideoCodecDRMPRIME* ctx = static_cast<CDVDVideoCodecDRMPRIME*>(avctx->opaque);
     auto buffer = dynamic_cast<CVideoBufferDMA*>(
-        ctx->m_processInfo.GetVideoBufferManager().Get(avctx->pix_fmt, size, nullptr));
+        ctx->m_processInfo.GetVideoBufferManager().Get(pix_fmt, size, nullptr));
     if (!buffer)
       return -1;
 
-    frame->opaque = static_cast<void*>(buffer);
     frame->opaque_ref =
-        av_buffer_create(nullptr, 0, ReleaseBuffer, frame->opaque, AV_BUFFER_FLAG_READONLY);
+        av_buffer_create(nullptr, 0, ReleaseBuffer, static_cast<void*>(buffer), AV_BUFFER_FLAG_READONLY);
 
     buffer->Export(frame, width, height);
     buffer->SyncStart();
@@ -635,9 +635,9 @@ bool CDVDVideoCodecDRMPRIME::SetPictureParams(VideoPicture* pVideoPicture)
     buffer->SetRef(m_pFrame);
     pVideoPicture->videoBuffer = buffer;
   }
-  else if (m_pFrame->opaque)
+  else if (IsSupportedSwFormat(static_cast<AVPixelFormat>(m_pFrame->format)))
   {
-    CVideoBufferDMA* buffer = static_cast<CVideoBufferDMA*>(m_pFrame->opaque);
+    CVideoBufferDMA* buffer = static_cast<CVideoBufferDMA*>(av_buffer_get_opaque(m_pFrame->buf[0]));
     buffer->SetPictureParams(*pVideoPicture);
     buffer->Acquire();
     buffer->SyncEnd();
@@ -671,13 +671,13 @@ void CDVDVideoCodecDRMPRIME::FilterTest()
 
     if (name.find("deinterlace") != std::string::npos)
     {
-      if (FilterOpen(name, true))
+      bool ret = FilterOpen(name, false, true);
+      FilterClose();
+      if (ret)
       {
         m_deintFilterName = name;
-
         CLog::Log(LOGDEBUG, "CDVDVideoCodecDRMPRIME::{} - found deinterlacing filter {}",
                   __FUNCTION__, name);
-
         return;
       }
     }
@@ -687,14 +687,31 @@ void CDVDVideoCodecDRMPRIME::FilterTest()
             __FUNCTION__);
 }
 
-bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
+AVFrame *CDVDVideoCodecDRMPRIME::alloc_filter_frame(AVFilterContext * ctx, void * v, int w, int h)
+{
+  int result;
+  CDVDVideoCodecDRMPRIME* me = static_cast<CDVDVideoCodecDRMPRIME*>(v);
+  AVFrame *frame = av_frame_alloc();
+  frame->width = w;
+  frame->height = h;
+  frame->format = AV_PIX_FMT_YUV420P;
+
+  if ((result = CDVDVideoCodecDRMPRIME::GetBuffer(me->m_pCodecContext, frame, 0)) < 0)
+  {
+    CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::alloc_filter_frame - failed to GetBuffer ({})", result);
+    return nullptr;
+  }
+  return frame;
+}
+
+bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool scale, bool test)
 {
   int result;
 
   if (m_pFilterGraph)
     FilterClose();
 
-  if (filters.empty())
+  if (filters.empty() && !scale)
     return true;
 
   if (!(m_pFilterGraph = avfilter_graph_alloc()))
@@ -705,13 +722,13 @@ bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
 
   const AVFilter* srcFilter = avfilter_get_by_name("buffer");
   const AVFilter* outFilter = avfilter_get_by_name("buffersink");
-  enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_DRM_PRIME, AV_PIX_FMT_NONE };
+  enum AVPixelFormat pix_fmts[] = { scale ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_DRM_PRIME, AV_PIX_FMT_NONE };
 
   std::string args = StringUtils::Format("video_size={}x{}:pix_fmt={}:time_base={}/{}:"
                                          "pixel_aspect={}/{}",
                                          m_pCodecContext->width,
                                          m_pCodecContext->height,
-                                         AV_PIX_FMT_DRM_PRIME,
+                                         scale ? m_pCodecContext->pix_fmt : AV_PIX_FMT_DRM_PRIME,
                                          m_pCodecContext->time_base.num ?
                                            m_pCodecContext->time_base.num : 1,
                                          m_pCodecContext->time_base.num ?
@@ -730,7 +747,6 @@ bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
     CLog::Log(LOGERROR,
               "CDVDVideoCodecDRMPRIME::FilterOpen - avfilter_graph_create_filter: src: {} ({})",
               err, result);
-    FilterClose();
     return false;
   }
 
@@ -738,7 +754,6 @@ bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
   if (!par)
   {
     CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::FilterOpen - unable to alloc buffersrc");
-    FilterClose();
     return false;
   }
 
@@ -754,7 +769,6 @@ bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
     CLog::Log(LOGERROR,
               "CDVDVideoCodecDRMPRIME::FilterOpen - av_buffersrc_parameters_set:  {} ({})",
               err, result);
-    FilterClose();
     return false;
   }
   av_freep(&par);
@@ -768,7 +782,6 @@ bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
     CLog::Log(LOGERROR,
               "CDVDVideoCodecDRMPRIME::FilterOpen - avfilter_graph_create_filter: out: {} ({})",
               err, result);
-    FilterClose();
     return false;
   }
 
@@ -777,32 +790,46 @@ bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
   if (result < 0)
   {
     CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::FilterOpen - failed settings pix formats");
-    FilterClose();
     return false;
   }
 
-  AVFilterInOut* outputs = avfilter_inout_alloc();
-  AVFilterInOut* inputs  = avfilter_inout_alloc();
+  if (!filters.empty())
+  {
+    AVFilterInOut* outputs = avfilter_inout_alloc();
+    AVFilterInOut* inputs  = avfilter_inout_alloc();
 
-  outputs->name = av_strdup("in");
-  outputs->filter_ctx = m_pFilterIn;
-  outputs->pad_idx = 0;
-  outputs->next = nullptr;
+    outputs->name = av_strdup("in");
+    outputs->filter_ctx = m_pFilterIn;
+    outputs->pad_idx = 0;
+    outputs->next = nullptr;
 
-  inputs->name = av_strdup("out");
-  inputs->filter_ctx = m_pFilterOut;
-  inputs->pad_idx = 0;
-  inputs->next = nullptr;
+    inputs->name = av_strdup("out");
+    inputs->filter_ctx = m_pFilterOut;
+    inputs->pad_idx = 0;
+    inputs->next = nullptr;
 
-  result = avfilter_graph_parse_ptr(m_pFilterGraph, filters.c_str(), &inputs, &outputs, NULL);
-  avfilter_inout_free(&outputs);
-  avfilter_inout_free(&inputs);
+    result = avfilter_graph_parse_ptr(m_pFilterGraph, filters.c_str(), &inputs, &outputs, NULL);
+    avfilter_inout_free(&outputs);
+    avfilter_inout_free(&inputs);
 
-  if (result < 0)
+    if (result < 0)
+    {
+      CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::FilterOpen - avfilter_graph_parse");
+      return false;
+    }
+  }
+  else
   {
-    CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::FilterOpen - avfilter_graph_parse");
-    FilterClose();
-    return false;
+    if ((result = av_buffersink_set_alloc_video_frame(m_pFilterOut, alloc_filter_frame, static_cast<void*>(this))) < 0)
+    {
+      CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::FilterOpen - av_buffersink_set_alloc_video_frame = {}", result);
+      return result;
+    }
+    if ((result = avfilter_link(m_pFilterIn, 0, m_pFilterOut, 0)) < 0)
+    {
+      CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::FilterOpen - avfilter_link");
+      return false;
+    }
   }
 
   if ((result = avfilter_graph_config(m_pFilterGraph,  nullptr)) < 0)
@@ -811,15 +838,11 @@ bool CDVDVideoCodecDRMPRIME::FilterOpen(const std::string& filters, bool test)
     av_strerror(result, err, AV_ERROR_MAX_STRING_SIZE);
     CLog::Log(LOGERROR, "CDVDVideoCodecDRMPRIME::FilterOpen - avfilter_graph_config:  {} ({})",
               err, result);
-    FilterClose();
     return false;
   }
 
   if (test)
-  {
-    FilterClose();
     return true;
-  }
 
   m_processInfo.SetVideoDeintMethod(filters);
 
@@ -854,16 +877,16 @@ void CDVDVideoCodecDRMPRIME::FilterClose()
 CDVDVideoCodec::VCReturn CDVDVideoCodecDRMPRIME::ProcessFilterIn()
 {
   // sw decoded buffers need cache flush and for descripter to be set
-  if (!IsSupportedHwFormat(static_cast<AVPixelFormat>(m_pFrame->format)) && m_pFrame->opaque != nullptr)
+  if (!IsSupportedHwFormat(static_cast<AVPixelFormat>(m_pFrame->format)) && IsSupportedSwFormat(static_cast<AVPixelFormat>(m_pFrame->format)))
   {
-    CVideoBufferDMA* buffer = static_cast<CVideoBufferDMA*>(m_pFrame->opaque);
+    CVideoBufferDMA* buffer = static_cast<CVideoBufferDMA*>(av_buffer_get_opaque(m_pFrame->buf[0]));
     buffer->SetDimensions(m_pFrame->width, m_pFrame->height);
     buffer->SyncEnd();
     auto descriptor = buffer->GetDescriptor();
     m_pFrame->data[0] = reinterpret_cast<uint8_t*>(descriptor);
+    m_pFrame->format = AV_PIX_FMT_DRM_PRIME;
   }
 
-  m_pFrame->format = AV_PIX_FMT_DRM_PRIME;
   int ret = av_buffersrc_add_frame(m_pFilterIn, m_pFrame);
   if (ret < 0)
   {
@@ -964,25 +987,28 @@ CDVDVideoCodec::VCReturn CDVDVideoCodecDRMPRIME::GetPicture(VideoPicture* pVideo
     return VC_ERROR;
   }
 
+  // we need to scale if the buffer isn't in DRM_PRIME format
+  bool need_scale = !IsSupportedSwFormat(static_cast<AVPixelFormat>(m_pFrame->format)) && !IsSupportedHwFormat(static_cast<AVPixelFormat>(m_pFrame->format));
+
   if (!m_processInfo.GetVideoInterlaced() && !!(m_pFrame->flags & AV_FRAME_FLAG_INTERLACED))
     m_processInfo.SetVideoInterlaced(true);
 
   std::string filterChain = GetFilterChain(!!(m_pFrame->flags & AV_FRAME_FLAG_INTERLACED));
-  if (!filterChain.empty())
+  if (!filterChain.empty() || need_scale)
   {
     bool reopenFilter = false;
     if (m_filters != filterChain)
       reopenFilter = true;
 
     if (m_pFilterGraph &&
-        (m_pFilterIn->outputs[0]->w != m_pCodecContext->width ||
-         m_pFilterIn->outputs[0]->h != m_pCodecContext->height))
+        (m_pFilterIn->outputs[0]->w != m_pFrame->width ||
+         m_pFilterIn->outputs[0]->h != m_pFrame->height))
       reopenFilter = true;
 
-    if (reopenFilter)
+    if (reopenFilter || (need_scale && m_pFilterGraph == nullptr))
     {
       m_filters = filterChain;
-      if (!FilterOpen(filterChain, false))
+      if (!FilterOpen(filterChain, need_scale, false))
         FilterClose();
     }
 
diff --git a/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.h b/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.h
index fab3431d40d9..bb88fde1f924 100644
--- a/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.h
+++ b/xbmc/cores/VideoPlayer/DVDCodecs/Video/DVDVideoCodecDRMPRIME.h
@@ -44,7 +44,8 @@ protected:
   CDVDVideoCodec::VCReturn ProcessFilterOut();
   static enum AVPixelFormat GetFormat(struct AVCodecContext* avctx, const enum AVPixelFormat* fmt);
   static int GetBuffer(struct AVCodecContext* avctx, AVFrame* frame, int flags);
-  bool FilterOpen(const std::string& filters, bool test);
+  static AVFrame *alloc_filter_frame(AVFilterContext * ctx, void * v, int w, int h);
+  bool FilterOpen(const std::string& filters, bool scale, bool test);
   void FilterClose();
   void FilterTest();
   std::string GetFilterChain(bool interlaced);
-- 
2.39.5

