diff --git a/README.md b/README.md index 8deb86a1..cf69c1cf 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,13 @@ are also available in the /docs/ source folder. * [Mac](https://github.com/OpenShot/libopenshot/wiki/Mac-Build-Instructions) * [Windows](https://github.com/OpenShot/libopenshot/wiki/Windows-Build-Instructions) +## Hardware Acceleration + +OpenShot now supports experimental hardware acceleration, both for encoding and +decoding videos. When enabled, this can either speed up those operations or slow +them down, depending on the power and features supported by your graphics card. +Please see [doc/HW-ACCELL.md](doc/HW-ACCEL.md) for more information. + ## Documentation Beautiful HTML documentation can be generated using Doxygen. diff --git a/cmake/Modules/FindFFmpeg.cmake b/cmake/Modules/FindFFmpeg.cmake index 34f0a7bd..c4eb7ca3 100644 --- a/cmake/Modules/FindFFmpeg.cmake +++ b/cmake/Modules/FindFFmpeg.cmake @@ -158,4 +158,4 @@ foreach (_component ${FFmpeg_FIND_COMPONENTS}) endforeach () # Give a nice error message if some of the required vars are missing. -find_package_handle_standard_args(FFmpeg DEFAULT_MSG ${_FFmpeg_REQUIRED_VARS}) \ No newline at end of file +find_package_handle_standard_args(FFmpeg DEFAULT_MSG ${_FFmpeg_REQUIRED_VARS}) diff --git a/doc/HW-ACCEL.md b/doc/HW-ACCEL.md new file mode 100644 index 00000000..b8ee7b4e --- /dev/null +++ b/doc/HW-ACCEL.md @@ -0,0 +1,134 @@ +## Hardware Acceleration + +OpenShot now has experimental support for hardware acceleration, which uses 1 (or more) +graphics cards to offload some of the work for both decoding and encoding. This is +very new and experimental (as of May 2019), but we look forward to "accelerating" +our support for this in the future! + +The following table summarizes our current level of support: + +| | Linux Decode | Linux Encode | Mac Decode | Mac Encode |Windows Decode| Windows Encode | Notes | +|--------------------|------------------------|----------------------|------------------|----------------|--------------|------------------|------------------| +| VA-API | :heavy_check_mark: | :heavy_check_mark: | - | - | - | - | *Linux Only* | +| VDPAU | :heavy_check_mark:(+) |:white_check_mark:(++)| - | - | - | - | *Linux Only* | +| CUDA (NVDEC/NVENC) | :x:(+++) | :heavy_check_mark: | - | - | - |:heavy_check_mark:| *Cross Platform* | +| VideoToolBox | - | - |:heavy_check_mark:| :x:(++++) | - | - | *Mac Only* | +| DXVA2 | - | - | - | - | :x:(+++) | - | *Windows Only* | +| D3D11VA | - | - | - | - | :x:(+++) | - | *Windows Only* | +| QSV | :x:(+++) | :x: | :x: | :x: | :x: | :x: | *Cross Platform* | + +* *(+) VDPAU for some reason needs a card number one higher than it really is* +* *(++) VDPAU is a decoder only.* +* *(+++) Green frames (pixel data not correctly tranferred back to system memory)* +* *(++++) Crashes and burns* + +## Supported FFmpeg Versions + +* HW accel is supported from FFmpeg version 3.2 (3.3 for nVidia drivers) +* HW accel was removed for nVidia drivers in Ubuntu for FFmpeg 4+ + +**Notice:** The FFmpeg versions of Ubuntu and PPAs for Ubuntu show the +same behaviour. FFmpeg 3 has working nVidia hardware acceleration while +FFmpeg 4+ has no support for nVidia hardware acceleration +included. + +## OpenShot Settings + +The following settings are use by libopenshot to enable, disable, and control +the various hardware acceleration features. + +``` +/// Use video codec for faster video decoding (if supported) +int HARDWARE_DECODER = 0; + +/* 0 - No acceleration + 1 - Linux VA-API + 2 - nVidia NVDEC + 3 - Windows D3D9 + 4 - Windows D3D11 + 5 - MacOS / VideoToolBox + 6 - Linux VDPAU + 7 - Intel QSV */ + +/// Number of threads of OpenMP +int OMP_THREADS = 12; + +/// Number of threads that FFmpeg uses +int FF_THREADS = 8; + +/// Maximum rows that hardware decode can handle +int DE_LIMIT_HEIGHT_MAX = 1100; + +/// Maximum columns that hardware decode can handle +int DE_LIMIT_WIDTH_MAX = 1950; + +/// Which GPU to use to decode (0 is the first, LINUX ONLY) +int HW_DE_DEVICE_SET = 0; + +/// Which GPU to use to encode (0 is the first, LINUX ONLY) +int HW_EN_DEVICE_SET = 0; +``` + +## Libva / VA-API (Video Acceleration API) + +The correct version of libva is needed (libva in Ubuntu 16.04 or libva2 +in Ubuntu 18.04) for the AppImage to work with hardware acceleration. +An AppImage that works on both systems (supporting libva and libva2), +might be possible when no libva is included in the AppImage. + +* vaapi is working for intel and AMD +* vaapi is working for decode only for nouveau +* nVidia driver is working for export only + +## AMD Graphics Cards (RadeonOpenCompute/ROCm) + +Decoding and encoding on the (AMD) GPU is possible with the default drivers. +On systems where ROCm is installed and run a future use for GPU acceleration +of effects could be implemented (contributions welcome). + +## Multiple Graphics Cards + +If the computer has multiple graphics cards installed, you can choose which +should be used by libopenshot. Also, you can optionally use one card for +decoding and the other for encoding (if both cards support acceleration). +This is currently only supported on Linux, due to the device name FFmpeg +expects (i.e. **/dev/dri/render128**). Contributions welcome if anyone can +determine what string format to pass for Windows and Mac. + +## Help Us Improve Hardware Support + +This information might be wrong, and we would love to continue improving +our support for hardware acceleration in OpenShot. Please help us update +this document if you find an error or discover new and/or useful information. + +**FFmpeg 4 + nVidia** The manual at: +https://www.tal.org/tutorials/ffmpeg_nvidia_encode +works pretty well. We could compile and install a version of FFmpeg 4.1.3 +on Mint 19.1 that supports the GPU on nVidia cards. A version of openshot +with hardware support using these libraries could use the nVidia GPU. + +**BUG:** Hardware supported decoding still has some bugs (as you can see from +the chart above). Also, the speed gains with decoding are not as great +as with encoding. Currently, if hardware decoding fails, there is no +fallback (you either get green frames or an "invalid file" error in OpenShot). +This needs to be improved to successfully fall-back to software decoding. + +**Needed:** + * A way to get options and limits of the GPU, such as + supported dimensions (width and height). + * A way to list the actual Graphic Cards available to FFmpeg (for the + user to choose which card for decoding and encoding, as opposed + to "Graphics Card X") + +**Further improvement:** Right now the frame can be decoded on the GPU, but the +frame is then copied to CPU memory for modifications. It is then copied back to +GPU memory for encoding. Using the GPU for both decoding and modifications +will make it possible to do away with these two copies. A possible solution would +be to use Vulkan compute which would be available on Linux and Windows natively +and on MacOS via MoltenVK. + +## Credit + +A big thanks to Peter M (https://github.com/eisneinechse) for all his work +on integrating hardware acceleration into libopenshot! The community thanks +you for this major contribution! diff --git a/include/AudioDeviceInfo.h b/include/AudioDeviceInfo.h new file mode 100644 index 00000000..29a89139 --- /dev/null +++ b/include/AudioDeviceInfo.h @@ -0,0 +1,43 @@ +/** + * @file + * @brief Header file for Audio Device Info struct + * @author Jonathan Thomas + * + * @section LICENSE + * + * Copyright (c) 2008-2014 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#ifndef OPENSHOT_AUDIODEVICEINFO_H +#define OPENSHOT_AUDIODEVICEINFO_H + + +/** + * @brief This struct hold information about Audio Devices + * + * The type and name of the audio device. + */ +struct AudioDeviceInfo +{ + string name; + string type; +}; + +#endif \ No newline at end of file diff --git a/include/FFmpegReader.h b/include/FFmpegReader.h index eaa45943..abf1af57 100644 --- a/include/FFmpegReader.h +++ b/include/FFmpegReader.h @@ -50,18 +50,17 @@ using namespace std; -namespace openshot -{ +namespace openshot { /** * @brief This struct holds the associated video frame and starting sample # for an audio packet. * * Because audio packets do not match up with video frames, this helps determine exactly * where the audio packet's samples belong. */ - struct AudioLocation - { + struct AudioLocation { int64_t frame; int sample_start; + bool is_near(AudioLocation location, int samples_per_frame, int64_t amount); }; @@ -91,14 +90,16 @@ namespace openshot * r.Close(); * @endcode */ - class FFmpegReader : public ReaderBase - { + class FFmpegReader : public ReaderBase { private: string path; AVFormatContext *pFormatCtx; int i, videoStream, audioStream; AVCodecContext *pCodecCtx, *aCodecCtx; +#if (LIBAVFORMAT_VERSION_MAJOR >= 57) + AVBufferRef *hw_device_ctx = NULL; //PM +#endif AVStream *pStream, *aStream; AVPacket *packet; AVFrame *pFrame; @@ -142,7 +143,15 @@ namespace openshot int64_t video_pts_offset; int64_t last_frame; int64_t largest_frame_processed; - int64_t current_video_frame; // can't reliably use PTS of video to determine this + int64_t current_video_frame; // can't reliably use PTS of video to determine this + + int hw_de_supported = 0; // Is set by FFmpegReader +#if IS_FFMPEG_3_2 + AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE; + AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE; +#endif + + int IsHardwareDecodeSupported(int codecid); /// Check for the correct frames per second value by scanning the 1st few seconds of video packets. void CheckFPS(); @@ -199,10 +208,10 @@ namespace openshot std::shared_ptr ReadStream(int64_t requested_frame); /// Remove AVFrame from cache (and deallocate it's memory) - void RemoveAVFrame(AVFrame*); + void RemoveAVFrame(AVFrame *); /// Remove AVPacket from cache (and deallocate it's memory) - void RemoveAVPacket(AVPacket*); + void RemoveAVPacket(AVPacket *); /// Seek to a specific Frame. This is not always frame accurate, it's more of an estimation on many codecs. void Seek(int64_t requested_frame); @@ -240,7 +249,7 @@ namespace openshot void Close(); /// Get the cache object used by this reader - CacheMemory* GetCache() { return &final_cache; }; + CacheMemory *GetCache() { return &final_cache; }; /// Get a shared pointer to a openshot::Frame object for a specific frame number of this reader. /// diff --git a/include/FFmpegUtilities.h b/include/FFmpegUtilities.h index 346da541..0d12ba72 100644 --- a/include/FFmpegUtilities.h +++ b/include/FFmpegUtilities.h @@ -42,6 +42,9 @@ extern "C" { #include #include + #if (LIBAVFORMAT_VERSION_MAJOR >= 57) + #include //PM + #endif #include // Change this to the first version swrescale works #if (LIBAVFORMAT_VERSION_MAJOR >= 57) @@ -114,6 +117,13 @@ #define PIX_FMT_YUV420P AV_PIX_FMT_YUV420P #endif + // FFmpeg's libavutil/common.h defines an RSHIFT incompatible with Ruby's + // definition in ruby/config.h, so we move it to FF_RSHIFT + #ifdef RSHIFT + #define FF_RSHIFT(a, b) RSHIFT(a, b) + #undef RSHIFT + #endif + #ifdef USE_SW #define SWR_CONVERT(ctx, out, linesize, out_count, in, linesize2, in_count) \ swr_convert(ctx, out, out_count, (const uint8_t **)in, in_count) diff --git a/include/FFmpegWriter.h b/include/FFmpegWriter.h index e219f72c..35dd1ed9 100644 --- a/include/FFmpegWriter.h +++ b/include/FFmpegWriter.h @@ -56,14 +56,12 @@ using namespace std; -namespace openshot -{ +namespace openshot { /// This enumeration designates the type of stream when encoding (video or audio) - enum StreamType - { - VIDEO_STREAM, ///< A video stream (used to determine which type of stream) - AUDIO_STREAM ///< An audio stream (used to determine which type of stream) + enum StreamType { + VIDEO_STREAM, ///< A video stream (used to determine which type of stream) + AUDIO_STREAM ///< An audio stream (used to determine which type of stream) }; /** @@ -141,8 +139,7 @@ namespace openshot * r.Close(); * @endcode */ - class FFmpegWriter : public WriterBase - { + class FFmpegWriter : public WriterBase { private: string path; int cache_size; @@ -155,56 +152,56 @@ namespace openshot bool write_header; bool write_trailer; - AVOutputFormat *fmt; - AVFormatContext *oc; - AVStream *audio_st, *video_st; - AVCodecContext *video_codec; - AVCodecContext *audio_codec; - SwsContext *img_convert_ctx; - double audio_pts, video_pts; - int16_t *samples; - uint8_t *audio_outbuf; - uint8_t *audio_encoder_buffer; + AVOutputFormat *fmt; + AVFormatContext *oc; + AVStream *audio_st, *video_st; + AVCodecContext *video_codec; + AVCodecContext *audio_codec; + SwsContext *img_convert_ctx; + double audio_pts, video_pts; + int16_t *samples; + uint8_t *audio_outbuf; + uint8_t *audio_encoder_buffer; - int num_of_rescalers; + int num_of_rescalers; int rescaler_position; - vector image_rescalers; + vector image_rescalers; - int audio_outbuf_size; - int audio_input_frame_size; - int initial_audio_input_frame_size; - int audio_input_position; - int audio_encoder_buffer_size; - SWRCONTEXT *avr; - SWRCONTEXT *avr_planar; + int audio_outbuf_size; + int audio_input_frame_size; + int initial_audio_input_frame_size; + int audio_input_position; + int audio_encoder_buffer_size; + SWRCONTEXT *avr; + SWRCONTEXT *avr_planar; - /* Resample options */ - int original_sample_rate; - int original_channels; + /* Resample options */ + int original_sample_rate; + int original_channels; - std::shared_ptr last_frame; - deque > spooled_audio_frames; - deque > spooled_video_frames; + std::shared_ptr last_frame; + deque > spooled_audio_frames; + deque > spooled_video_frames; - deque > queued_audio_frames; - deque > queued_video_frames; + deque > queued_audio_frames; + deque > queued_video_frames; - deque > processed_frames; - deque > deallocate_frames; + deque > processed_frames; + deque > deallocate_frames; - map, AVFrame*> av_frames; + map, AVFrame *> av_frames; - /// Add an AVFrame to the cache - void add_avframe(std::shared_ptr frame, AVFrame* av_frame); + /// Add an AVFrame to the cache + void add_avframe(std::shared_ptr frame, AVFrame *av_frame); /// Add an audio output stream - AVStream* add_audio_stream(); + AVStream *add_audio_stream(); /// Add a video output stream - AVStream* add_video_stream(); + AVStream *add_video_stream(); /// Allocate an AVFrame object - AVFrame* allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer); + AVFrame *allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer); /// Auto detect format (from path) void auto_detect_format(); @@ -236,10 +233,10 @@ namespace openshot void process_video_packet(std::shared_ptr frame); /// write all queued frames' audio to the video file - void write_audio_packets(bool final); + void write_audio_packets(bool is_final); /// write video frame - bool write_video_packet(std::shared_ptr frame, AVFrame* frame_final); + bool write_video_packet(std::shared_ptr frame, AVFrame *frame_final); /// write all queued frames void write_queued_frames(); @@ -303,7 +300,7 @@ namespace openshot /// @param interlaced Does this video need to be interlaced? /// @param top_field_first Which frame should be used as the top field? /// @param bit_rate The video bit rate used during encoding - void SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height,Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate); + void SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate); /// @brief Set custom options (some codecs accept additional params). This must be called after the /// PrepareStreams() method, otherwise the streams have not been initialized yet. @@ -324,7 +321,7 @@ namespace openshot /// @param reader A openshot::ReaderBase object which will provide frames to be written /// @param start The starting frame number of the reader /// @param length The number of frames to write - void WriteFrame(ReaderBase* reader, int64_t start, int64_t length); + void WriteFrame(ReaderBase *reader, int64_t start, int64_t length); /// @brief Write the file trailer (after all frames are written). This is called automatically /// by the Close() method if this method has not yet been called. diff --git a/include/OpenMPUtilities.h b/include/OpenMPUtilities.h index 65047c31..7c198a76 100644 --- a/include/OpenMPUtilities.h +++ b/include/OpenMPUtilities.h @@ -32,11 +32,14 @@ #include #include -// Calculate the # of OpenMP and FFmpeg Threads to allow. We are limiting both -// of these based on our own performance tests (more is not always better). -#define OPEN_MP_NUM_PROCESSORS (min(omp_get_num_procs(), 6)) -#define FF_NUM_PROCESSORS (min(omp_get_num_procs(), 12)) +#include "../include/Settings.h" +using namespace std; +using namespace openshot; + +// Calculate the # of OpenMP Threads to allow +#define OPEN_MP_NUM_PROCESSORS (min(omp_get_num_procs(), max(2, openshot::Settings::Instance()->OMP_THREADS) )) +#define FF_NUM_PROCESSORS (min(omp_get_num_procs(), max(2, openshot::Settings::Instance()->FF_THREADS) )) #endif diff --git a/include/Qt/AudioPlaybackThread.h b/include/Qt/AudioPlaybackThread.h index 1d654756..be26c4e8 100644 --- a/include/Qt/AudioPlaybackThread.h +++ b/include/Qt/AudioPlaybackThread.h @@ -32,6 +32,8 @@ #include "../ReaderBase.h" #include "../RendererBase.h" #include "../AudioReaderSource.h" +#include "../AudioDeviceInfo.h" +#include "../Settings.h" namespace openshot { @@ -66,8 +68,11 @@ namespace openshot /// Error found during JUCE initialise method string initialise_error; - /// Create or get an instance of this singleton (invoke the class with this method) - static AudioDeviceManagerSingleton * Instance(int numChannels); + /// List of valid audio device names + vector audio_device_names; + + /// Override with no channels and no preferred audio device + static AudioDeviceManagerSingleton * Instance(); /// Public device manager property AudioDeviceManager audioDeviceManager; @@ -126,7 +131,10 @@ namespace openshot int getSpeed() const { if (source) return source->getSpeed(); else return 1; } /// Get Audio Error (if any) - string getError() { return AudioDeviceManagerSingleton::Instance(numChannels)->initialise_error; } + string getError() { return AudioDeviceManagerSingleton::Instance()->initialise_error; } + + /// Get Audio Device Names (if any) + vector getAudioDeviceNames() { return AudioDeviceManagerSingleton::Instance()->audio_device_names; }; friend class PlayerPrivate; friend class QtPlayer; diff --git a/include/QtImageReader.h b/include/QtImageReader.h index 6b260f15..e4d14f9b 100644 --- a/include/QtImageReader.h +++ b/include/QtImageReader.h @@ -65,9 +65,10 @@ namespace openshot { private: string path; - std::shared_ptr image; ///> Original image (full quality) - std::shared_ptr cached_image; ///> Scaled for performance - bool is_open; + std::shared_ptr image; ///> Original image (full quality) + std::shared_ptr cached_image; ///> Scaled for performance + bool is_open; ///> Is Reader opened + QSize max_size; ///> Current max_size as calculated with Clip properties public: diff --git a/include/QtPlayer.h b/include/QtPlayer.h index a1a7ee0c..c9137f5e 100644 --- a/include/QtPlayer.h +++ b/include/QtPlayer.h @@ -62,6 +62,9 @@ namespace openshot /// Get Error (if any) string GetError(); + /// Get Audio Devices from JUCE + vector GetAudioDeviceNames(); + /// Play the video void Play(); diff --git a/include/Settings.h b/include/Settings.h index e46f12e0..89561034 100644 --- a/include/Settings.h +++ b/include/Settings.h @@ -76,11 +76,19 @@ namespace openshot { static Settings * m_pInstance; public: - /// Use video card for faster video decoding (if supported) - bool HARDWARE_DECODE = false; - - /// Use video card for faster video encoding (if supported) - bool HARDWARE_ENCODE = false; + /** + * @brief Use video codec for faster video decoding (if supported) + * + * 0 - No acceleration, + * 1 - Linux VA-API, + * 2 - nVidia NVDEC, + * 3 - Windows D3D9, + * 4 - Windows D3D11, + * 5 - MacOS / VideoToolBox, + * 6 - Linux VDPAU, + * 7 - Intel QSV + */ + int HARDWARE_DECODER = 0; /// Scale mode used in FFmpeg decoding and encoding (used as an optimization for faster previews) bool HIGH_QUALITY_SCALING = false; @@ -94,6 +102,27 @@ namespace openshot { /// Wait for OpenMP task to finish before continuing (used to limit threads on slower systems) bool WAIT_FOR_VIDEO_PROCESSING_TASK = false; + /// Number of threads of OpenMP + int OMP_THREADS = 12; + + /// Number of threads that ffmpeg uses + int FF_THREADS = 8; + + /// Maximum rows that hardware decode can handle + int DE_LIMIT_HEIGHT_MAX = 1100; + + /// Maximum columns that hardware decode can handle + int DE_LIMIT_WIDTH_MAX = 1950; + + /// Which GPU to use to decode (0 is the first) + int HW_DE_DEVICE_SET = 0; + + /// Which GPU to use to encode (0 is the first) + int HW_EN_DEVICE_SET = 0; + + /// The audio device name to use during playback + string PLAYBACK_AUDIO_DEVICE_NAME = ""; + /// Create or get an instance of this logger singleton (invoke the class with this method) static Settings * Instance(); }; diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index a8d1d746..acd3b55f 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -30,15 +30,63 @@ #include "../include/FFmpegReader.h" +#define ENABLE_VAAPI 0 + +#if IS_FFMPEG_3_2 +#pragma message "You are compiling with experimental hardware decode" +#else +#pragma message "You are compiling only with software decode" +#endif + +#if IS_FFMPEG_3_2 +#define MAX_SUPPORTED_WIDTH 1950 +#define MAX_SUPPORTED_HEIGHT 1100 + +#if ENABLE_VAAPI +#include "libavutil/hwcontext_vaapi.h" + +typedef struct VAAPIDecodeContext { + VAProfile va_profile; + VAEntrypoint va_entrypoint; + VAConfigID va_config; + VAContextID va_context; + +#if FF_API_STRUCT_VAAPI_CONTEXT + // FF_DISABLE_DEPRECATION_WARNINGS + int have_old_context; + struct vaapi_context *old_context; + AVBufferRef *device_ref; + // FF_ENABLE_DEPRECATION_WARNINGS +#endif + + AVHWDeviceContext *device; + AVVAAPIDeviceContext *hwctx; + + AVHWFramesContext *frames; + AVVAAPIFramesContext *hwfc; + + enum AVPixelFormat surface_format; + int surface_count; + } VAAPIDecodeContext; +#endif +#endif + + using namespace openshot; +int hw_de_on = 0; +#if IS_FFMPEG_3_2 + AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE; + AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE; +#endif + FFmpegReader::FFmpegReader(string path) - : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0), - audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false), - check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0), - prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0), - current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), - packet(NULL) { + : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0), + audio_pts_offset(99999), video_pts_offset(99999), path(path), is_video_seek(true), check_interlace(false), + check_fps(false), enable_seek(true), is_open(false), seek_audio_frame_found(0), seek_video_frame_found(0), + prev_samples(0), prev_pts(0), pts_total(0), pts_counter(0), is_duration_known(false), largest_frame_processed(0), + current_video_frame(0), has_missing_frames(false), num_packets_since_video_frame(0), num_checks_since_final(0), + packet(NULL) { // Initialize FFMpeg, and register all formats and codecs AV_REGISTER_ALL @@ -85,8 +133,7 @@ FFmpegReader::~FFmpegReader() { } // This struct holds the associated video frame and starting sample # for an audio packet. -bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) -{ +bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64_t amount) { // Is frame even close to this one? if (abs(location.frame - frame) >= 2) // This is too far away to be considered @@ -103,13 +150,94 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64 return false; } -void FFmpegReader::Open() +#if IS_FFMPEG_3_2 + +// Get hardware pix format +static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) { + const enum AVPixelFormat *p; + + for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { + switch (*p) { +#if defined(__linux__) + // Linux pix formats + case AV_PIX_FMT_VAAPI: + hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI; + return *p; + break; + case AV_PIX_FMT_VDPAU: + hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU; + return *p; + break; +#endif +#if defined(_WIN32) + // Windows pix formats + case AV_PIX_FMT_DXVA2_VLD: + hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2; + return *p; + break; + case AV_PIX_FMT_D3D11: + hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA; + return *p; + break; +#endif +#if defined(__APPLE__) + // Apple pix formats + case AV_PIX_FMT_VIDEOTOOLBOX: + hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; + return *p; + break; +#endif + // Cross-platform pix formats + case AV_PIX_FMT_CUDA: + hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA; + return *p; + break; + case AV_PIX_FMT_QSV: + hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV; + return *p; + break; + } + } + ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::get_hw_dec_format (Unable to decode this file using hardware decode)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + return AV_PIX_FMT_NONE; +} + +int FFmpegReader::IsHardwareDecodeSupported(int codecid) +{ + int ret; + switch (codecid) { + case AV_CODEC_ID_H264: + case AV_CODEC_ID_MPEG2VIDEO: + case AV_CODEC_ID_VC1: + case AV_CODEC_ID_WMV1: + case AV_CODEC_ID_WMV2: + case AV_CODEC_ID_WMV3: + ret = 1; + break; + default : + ret = 0; + break; + } + return ret; +} +#endif + +void FFmpegReader::Open() { // Open reader if not already open - if (!is_open) - { + if (!is_open) { // Initialize format context pFormatCtx = NULL; + { + hw_de_on = (openshot::Settings::Instance()->HARDWARE_DECODER == 0 ? 0 : 1); + } // Open video file if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0) @@ -122,8 +250,7 @@ void FFmpegReader::Open() videoStream = -1; audioStream = -1; // Loop through each stream, and identify the video and audio stream index - for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) - { + for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) { // Is this a video stream? if (AV_GET_CODEC_TYPE(pFormatCtx->streams[i]) == AVMEDIA_TYPE_VIDEO && videoStream < 0) { videoStream = i; @@ -137,8 +264,7 @@ void FFmpegReader::Open() throw NoStreamsFound("No video or audio streams found in this file.", path); // Is there a video stream? - if (videoStream != -1) - { + if (videoStream != -1) { // Set the stream index info.video_stream_index = videoStream; @@ -150,23 +276,232 @@ void FFmpegReader::Open() // Get codec and codec context from stream AVCodec *pCodec = avcodec_find_decoder(codecId); - pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec); - - // Set number of threads equal to number of processors (not to exceed 16) - pCodecCtx->thread_count = min(FF_NUM_PROCESSORS, 16); - - if (pCodec == NULL) { - throw InvalidCodec("A valid video codec could not be found for this file.", path); - } - - // Init options AVDictionary *opts = NULL; - av_dict_set(&opts, "strict", "experimental", 0); + int retry_decode_open = 2; + // If hw accel is selected but hardware cannot handle repeat with software decoding + do { + pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec); +#if IS_FFMPEG_3_2 + if (hw_de_on && (retry_decode_open==2)) { + // Up to here no decision is made if hardware or software decode + hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id); + } +#endif + retry_decode_open = 0; - // Open video codec - if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0) - throw InvalidCodec("A video codec was found, but could not be opened.", path); + // Set number of threads equal to number of processors (not to exceed 16) + pCodecCtx->thread_count = min(FF_NUM_PROCESSORS, 16); + if (pCodec == NULL) { + throw InvalidCodec("A valid video codec could not be found for this file.", path); + } + + // Init options + av_dict_set(&opts, "strict", "experimental", 0); +#if IS_FFMPEG_3_2 + if (hw_de_on && hw_de_supported) { + // Open Hardware Acceleration + int i_decoder_hw = 0; + char adapter[256]; + char *adapter_ptr = NULL; + int adapter_num; + adapter_num = openshot::Settings::Instance()->HW_DE_DEVICE_SET; + fprintf(stderr, "\n\nDecodiing Device Nr: %d\n", adapter_num); + + // Set hardware pix format (callback) + pCodecCtx->get_format = get_hw_dec_format; + + if (adapter_num < 3 && adapter_num >=0) { +#if defined(__linux__) + snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128); + adapter_ptr = adapter; + i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER; + switch (i_decoder_hw) { + case 1: + hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI; + break; + case 2: + hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA; + break; + case 6: + hw_de_av_device_type = AV_HWDEVICE_TYPE_VDPAU; + break; + case 7: + hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; + break; + default: + hw_de_av_device_type = AV_HWDEVICE_TYPE_VAAPI; + break; + } + +#elif defined(_WIN32) + adapter_ptr = NULL; + i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER; + switch (i_decoder_hw) { + case 2: + hw_de_av_device_type = AV_HWDEVICE_TYPE_CUDA; + break; + case 3: + hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2; + break; + case 4: + hw_de_av_device_type = AV_HWDEVICE_TYPE_D3D11VA; + break; + case 7: + hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; + break; + default: + hw_de_av_device_type = AV_HWDEVICE_TYPE_DXVA2; + break; + } +#elif defined(__APPLE__) + adapter_ptr = NULL; + i_decoder_hw = openshot::Settings::Instance()->HARDWARE_DECODER; + switch (i_decoder_hw) { + case 5: + hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; + break; + case 7: + hw_de_av_device_type = AV_HWDEVICE_TYPE_QSV; + break; + default: + hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; + break; + } +#endif + + } else { + adapter_ptr = NULL; // Just to be sure + } + + // Check if it is there and writable +#if defined(__linux__) + if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) { +#elif defined(_WIN32) + if( adapter_ptr != NULL ) { +#elif defined(__APPLE__) + if( adapter_ptr != NULL ) { +#endif + ZmqLogger::Instance()->AppendDebugMethod("Decode Device present using device", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } + else { + adapter_ptr = NULL; // use default + ZmqLogger::Instance()->AppendDebugMethod("Decode Device not present using default", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } + + hw_device_ctx = NULL; + // Here the first hardware initialisations are made + if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { + if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) { + throw InvalidCodec("Hardware device reference create failed.", path); + } + + /* + av_buffer_unref(&ist->hw_frames_ctx); + ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx); + if (!ist->hw_frames_ctx) { + av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n"); + return AVERROR(ENOMEM); + } + + frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data; + + frames_ctx->format = AV_PIX_FMT_CUDA; + frames_ctx->sw_format = avctx->sw_pix_fmt; + frames_ctx->width = avctx->width; + frames_ctx->height = avctx->height; + + av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n", + av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height); + + + ret = av_hwframe_ctx_init(pCodecCtx->hw_device_ctx); + ret = av_hwframe_ctx_init(ist->hw_frames_ctx); + if (ret < 0) { + av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n"); + return ret; + } + */ + } + else { + throw InvalidCodec("Hardware device create failed.", path); + } + } +#endif + + // Open video codec + if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0) + throw InvalidCodec("A video codec was found, but could not be opened.", path); + +#if IS_FFMPEG_3_2 + if (hw_de_on && hw_de_supported) { + AVHWFramesConstraints *constraints = NULL; + void *hwconfig = NULL; + hwconfig = av_hwdevice_hwconfig_alloc(hw_device_ctx); + +// TODO: needs va_config! +#if ENABLE_VAAPI + ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config; +#endif + constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig); + if (constraints) { + if (pCodecCtx->coded_width < constraints->min_width || + pCodecCtx->coded_height < constraints->min_height || + pCodecCtx->coded_width > constraints->max_width || + pCodecCtx->coded_height > constraints->max_height) { + ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + hw_de_supported = 0; + retry_decode_open = 1; + AV_FREE_CONTEXT(pCodecCtx); + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + hw_device_ctx = NULL; + } + } + else { + // All is just peachy + ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Min width :", constraints->min_width, "Min Height :", constraints->min_height, "MaxWidth :", constraints->max_width, "MaxHeight :", constraints->max_height, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height); + retry_decode_open = 0; + } + av_hwframe_constraints_free(&constraints); + if (hwconfig) { + av_freep(&hwconfig); + } + } + else { + int max_h, max_w; + //max_h = ((getenv( "LIMIT_HEIGHT_MAX" )==NULL) ? MAX_SUPPORTED_HEIGHT : atoi(getenv( "LIMIT_HEIGHT_MAX" ))); + max_h = openshot::Settings::Instance()->DE_LIMIT_HEIGHT_MAX; + //max_w = ((getenv( "LIMIT_WIDTH_MAX" )==NULL) ? MAX_SUPPORTED_WIDTH : atoi(getenv( "LIMIT_WIDTH_MAX" ))); + max_w = openshot::Settings::Instance()->DE_LIMIT_WIDTH_MAX; + ZmqLogger::Instance()->AppendDebugMethod("Constraints could not be found using default limit\n", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + //cerr << "Constraints could not be found using default limit\n"; + if (pCodecCtx->coded_width < 0 || + pCodecCtx->coded_height < 0 || + pCodecCtx->coded_width > max_w || + pCodecCtx->coded_height > max_h ) { + ZmqLogger::Instance()->AppendDebugMethod("DIMENSIONS ARE TOO LARGE for hardware acceleration\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height, "", -1, "", -1); + hw_de_supported = 0; + retry_decode_open = 1; + AV_FREE_CONTEXT(pCodecCtx); + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + hw_device_ctx = NULL; + } + } + else { + ZmqLogger::Instance()->AppendDebugMethod("\nDecode hardware acceleration is used\n", "Max Width :", max_w, "Max Height :", max_h, "Frame width :", pCodecCtx->coded_width, "Frame height :", pCodecCtx->coded_height, "", -1, "", -1); + retry_decode_open = 0; + } + } + } // if hw_de_on && hw_de_supported + else { + ZmqLogger::Instance()->AppendDebugMethod("\nDecode in software is used\n", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } +#else + retry_decode_open = 0; +#endif + } while (retry_decode_open); // retry_decode_open // Free options av_dict_free(&opts); @@ -175,8 +510,7 @@ void FFmpegReader::Open() } // Is there an audio stream? - if (audioStream != -1) - { + if (audioStream != -1) { // Set the stream index info.audio_stream_index = audioStream; @@ -234,24 +568,28 @@ void FFmpegReader::Open() } } -void FFmpegReader::Close() -{ +void FFmpegReader::Close() { // Close all objects, if reader is 'open' - if (is_open) - { + if (is_open) { // Mark as "closed" is_open = false; ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::Close", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); // Close the codec - if (info.has_video) - { + if (info.has_video) { avcodec_flush_buffers(pCodecCtx); AV_FREE_CONTEXT(pCodecCtx); +#if IS_FFMPEG_3_2 + if (hw_de_on) { + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + hw_device_ctx = NULL; + } + } +#endif } - if (info.has_audio) - { + if (info.has_audio) { avcodec_flush_buffers(aCodecCtx); AV_FREE_CONTEXT(aCodecCtx); } @@ -263,7 +601,7 @@ void FFmpegReader::Close() // Clear processed lists { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processed_video_frames.clear(); processed_audio_frames.clear(); processing_video_frames.clear(); @@ -289,15 +627,14 @@ void FFmpegReader::Close() } } -void FFmpegReader::UpdateAudioInfo() -{ +void FFmpegReader::UpdateAudioInfo() { // Set values of FileInfo struct info.has_audio = true; info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1; info.acodec = aCodecCtx->codec->name; info.channels = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels; if (AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout == 0) - AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout( AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels ); + AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout = av_get_default_channel_layout(AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels); info.channel_layout = (ChannelLayout) AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout; info.sample_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->sample_rate; info.audio_bit_rate = AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->bit_rate; @@ -311,15 +648,13 @@ void FFmpegReader::UpdateAudioInfo() info.duration = aStream->duration * info.audio_timebase.ToDouble(); // Check for an invalid video length - if (info.has_video && info.video_length <= 0) - { + if (info.has_video && info.video_length <= 0) { // Calculate the video length from the audio duration info.video_length = info.duration * info.fps.ToDouble(); } // Set video timebase (if no video stream was found) - if (!info.has_video) - { + if (!info.has_video) { // Set a few important default video settings (so audio can be divided into frames) info.fps.num = 24; info.fps.den = 1; @@ -344,8 +679,7 @@ void FFmpegReader::UpdateAudioInfo() } } -void FFmpegReader::UpdateVideoInfo() -{ +void FFmpegReader::UpdateVideoInfo() { if (check_fps) // Already initialized all the video metadata, no reason to do it again return; @@ -362,18 +696,13 @@ void FFmpegReader::UpdateVideoInfo() info.fps.num = pStream->avg_frame_rate.num; info.fps.den = pStream->avg_frame_rate.den; - if (pStream->sample_aspect_ratio.num != 0) - { + if (pStream->sample_aspect_ratio.num != 0) { info.pixel_ratio.num = pStream->sample_aspect_ratio.num; info.pixel_ratio.den = pStream->sample_aspect_ratio.den; - } - else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) - { + } else if (AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num != 0) { info.pixel_ratio.num = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.num; info.pixel_ratio.den = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->sample_aspect_ratio.den; - } - else - { + } else { info.pixel_ratio.num = 1; info.pixel_ratio.den = 1; } @@ -407,15 +736,12 @@ void FFmpegReader::UpdateVideoInfo() info.duration = (info.file_size / info.video_bit_rate); // No duration found in stream of file - if (info.duration <= 0.0f) - { + if (info.duration <= 0.0f) { // No duration is found in the video stream info.duration = -1; info.video_length = -1; is_duration_known = false; - } - else - { + } else { // Yes, a duration was found is_duration_known = true; @@ -440,8 +766,7 @@ void FFmpegReader::UpdateVideoInfo() } -std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) -{ +std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { // Check for open reader (or throw exception) if (!is_open) throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path); @@ -466,11 +791,9 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) // Return the cached frame return frame; - } - else - { - #pragma omp critical (ReadStream) - { + } else { +#pragma omp critical (ReadStream) + { // Check the cache a 2nd time (due to a potential previous lock) frame = final_cache.GetFrame(requested_frame); if (frame) { @@ -478,8 +801,7 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame, "", -1, "", -1, "", -1, "", -1, "", -1); // Return the cached frame - } - else { + } else { // Frame is not in cache // Reset seek count seek_count = 0; @@ -491,20 +813,16 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) // Are we within X frames of the requested frame? int64_t diff = requested_frame - last_frame; - if (diff >= 1 && diff <= 20) - { + if (diff >= 1 && diff <= 20) { // Continue walking the stream frame = ReadStream(requested_frame); - } - else - { + } else { // Greater than 30 frames away, or backwards, we need to seek to the nearest key frame if (enable_seek) // Only seek if enabled Seek(requested_frame); - else if (!enable_seek && diff < 0) - { + else if (!enable_seek && diff < 0) { // Start over, since we can't seek, and the requested frame is smaller than our position Close(); Open(); @@ -514,14 +832,13 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) frame = ReadStream(requested_frame); } } - } //omp critical - return frame; + } //omp critical + return frame; } } // Read the stream until we find the requested Frame -std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) -{ +std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { // Allocate video frame bool end_of_stream = false; bool check_seek = false; @@ -531,7 +848,7 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) // Minimum number of packets to process (for performance reasons) int packets_processed = 0; int minimum_packets = OPEN_MP_NUM_PROCESSORS; - int max_packets = 4096; + int max_packets = 4096; // Set the number of threads in OpenMP omp_set_num_threads(OPEN_MP_NUM_PROCESSORS); @@ -541,20 +858,19 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream", "requested_frame", requested_frame, "OPEN_MP_NUM_PROCESSORS", OPEN_MP_NUM_PROCESSORS, "", -1, "", -1, "", -1, "", -1); - #pragma omp parallel +#pragma omp parallel { - #pragma omp single +#pragma omp single { // Loop through the stream until the correct frame is found - while (true) - { + while (true) { // Get the next packet into a local variable called packet packet_error = GetNextPacket(); int processing_video_frames_size = 0; int processing_audio_frames_size = 0; { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_video_frames_size = processing_video_frames.size(); processing_audio_frames_size = processing_audio_frames.size(); } @@ -562,14 +878,13 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) // Wait if too many frames are being processed while (processing_video_frames_size + processing_audio_frames_size >= minimum_packets) { usleep(2500); - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_video_frames_size = processing_video_frames.size(); processing_audio_frames_size = processing_audio_frames.size(); } // Get the next packet (if any) - if (packet_error < 0) - { + if (packet_error < 0) { // Break loop when no more packets found end_of_stream = true; break; @@ -579,29 +894,27 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ReadStream (GetNextPacket)", "requested_frame", requested_frame, "processing_video_frames_size", processing_video_frames_size, "processing_audio_frames_size", processing_audio_frames_size, "minimum_packets", minimum_packets, "packets_processed", packets_processed, "is_seeking", is_seeking); // Video packet - if (info.has_video && packet->stream_index == videoStream) - { + if (info.has_video && packet->stream_index == videoStream) { // Reset this counter, since we have a video packet num_packets_since_video_frame = 0; // Check the status of a seek (if any) - if (is_seeking) - #pragma omp critical (openshot_seek) - check_seek = CheckSeek(true); - else - check_seek = false; + if (is_seeking) +#pragma omp critical (openshot_seek) + check_seek = CheckSeek(true); + else + check_seek = false; - if (check_seek) { - // Jump to the next iteration of this loop - continue; - } + if (check_seek) { + // Jump to the next iteration of this loop + continue; + } // Get the AVFrame from the current packet frame_finished = GetAVFrame(); // Check if the AVFrame is finished and set it - if (frame_finished) - { + if (frame_finished) { // Update PTS / Frame Offset (if any) UpdatePTSOffset(true); @@ -611,20 +924,19 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) if (openshot::Settings::Instance()->WAIT_FOR_VIDEO_PROCESSING_TASK) { // Wait on each OMP task to complete before moving on to the next one. This slows // down processing considerably, but might be more stable on some systems. - #pragma omp taskwait +#pragma omp taskwait } } } // Audio packet - else if (info.has_audio && packet->stream_index == audioStream) - { + else if (info.has_audio && packet->stream_index == audioStream) { // Increment this (to track # of packets since the last video packet) num_packets_since_video_frame++; // Check the status of a seek (if any) if (is_seeking) - #pragma omp critical (openshot_seek) +#pragma omp critical (openshot_seek) check_seek = CheckSeek(false); else check_seek = false; @@ -686,8 +998,7 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) if (frame) { // return the largest processed frame (assuming it was the last in the video file) return frame; - } - else { + } else { // The largest processed frame is no longer in cache, return a blank frame std::shared_ptr f = CreateFrame(largest_frame_processed); f->AddColor(info.width, info.height, "#000"); @@ -698,51 +1009,86 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) } // Get the next packet (if any) -int FFmpegReader::GetNextPacket() -{ +int FFmpegReader::GetNextPacket() { int found_packet = 0; - AVPacket *next_packet = new AVPacket(); - found_packet = av_read_frame(pFormatCtx, next_packet); - - if (packet) { - // Remove previous packet before getting next one - RemoveAVPacket(packet); - packet = NULL; - } - - if (found_packet >= 0) + AVPacket *next_packet; +#pragma omp critical(getnextpacket) { - // Update current packet pointer - packet = next_packet; - } + next_packet = new AVPacket(); + found_packet = av_read_frame(pFormatCtx, next_packet); + + if (packet) { + // Remove previous packet before getting next one + RemoveAVPacket(packet); + packet = NULL; + } + + if (found_packet >= 0) { + // Update current packet pointer + packet = next_packet; + } + } // Return if packet was found (or error number) return found_packet; } // Get an AVFrame (if any) -bool FFmpegReader::GetAVFrame() -{ +bool FFmpegReader::GetAVFrame() { int frameFinished = -1; int ret = 0; // Decode video frame AVFrame *next_frame = AV_ALLOCATE_FRAME(); - #pragma omp critical (packet_cache) +#pragma omp critical (packet_cache) { - #if IS_FFMPEG_3_2 +#if IS_FFMPEG_3_2 frameFinished = 0; + ret = avcodec_send_packet(pCodecCtx, packet); + + // Get the format from the variables set in get_hw_dec_format + hw_de_av_pix_fmt = hw_de_av_pix_fmt_global; + hw_de_av_device_type = hw_de_av_device_type_global; + if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); } else { + AVFrame *next_frame2; + if (hw_de_on && hw_de_supported) { + next_frame2 = AV_ALLOCATE_FRAME(); + } + else + { + next_frame2 = next_frame; + } pFrame = new AVFrame(); while (ret >= 0) { - ret = avcodec_receive_frame(pCodecCtx, next_frame); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { - break; + ret = avcodec_receive_frame(pCodecCtx, next_frame2); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; } + if (ret != 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (invalid return frame received)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } + if (hw_de_on && hw_de_supported) { + int err; + if (next_frame2->format == hw_de_av_pix_fmt) { + next_frame->format = AV_PIX_FMT_YUV420P; + if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } + if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } + } + } + else + { // No hardware acceleration used -> no copy from GPU memory needed + next_frame = next_frame2; + } + // TODO also handle possible further frames // Use only the first frame like avcodec_decode_video2 if (frameFinished == 0 ) { @@ -757,8 +1103,11 @@ bool FFmpegReader::GetAVFrame() } } } + if (hw_de_on && hw_de_supported) { + AV_FREE_FRAME(&next_frame2); + } } - #else +#else avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet); // is frame finished @@ -777,7 +1126,7 @@ bool FFmpegReader::GetAVFrame() info.top_field_first = next_frame->top_field_first; } } - #endif +#endif } // deallocate the frame @@ -788,11 +1137,9 @@ bool FFmpegReader::GetAVFrame() } // Check the current seek position and determine if we need to seek again -bool FFmpegReader::CheckSeek(bool is_video) -{ +bool FFmpegReader::CheckSeek(bool is_video) { // Are we seeking for a specific frame? - if (is_seeking) - { + if (is_seeking) { // Determine if both an audio and video packet have been decoded since the seek happened. // If not, allow the ReadStream method to keep looping if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found)) @@ -808,16 +1155,13 @@ bool FFmpegReader::CheckSeek(bool is_video) max_seeked_frame = seek_video_frame_found; // determine if we are "before" the requested frame - if (max_seeked_frame >= seeking_frame) - { + if (max_seeked_frame >= seeking_frame) { // SEEKED TOO FAR ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Too far, seek again)", "is_video_seek", is_video_seek, "max_seeked_frame", max_seeked_frame, "seeking_frame", seeking_frame, "seeking_pts", seeking_pts, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found); // Seek again... to the nearest Keyframe Seek(seeking_frame - (10 * seek_count * seek_count)); - } - else - { + } else { // SEEK WORKED ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", "is_video_seek", is_video_seek, "current_pts", packet->pts, "seeking_pts", seeking_pts, "seeking_frame", seeking_frame, "seek_video_frame_found", seek_video_frame_found, "seek_audio_frame_found", seek_audio_frame_found); @@ -833,8 +1177,7 @@ bool FFmpegReader::CheckSeek(bool is_video) } // Process a video packet -void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) -{ +void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { // Calculate current frame # int64_t current_frame = ConvertVideoPTStoFrame(GetVideoPTS()); @@ -843,8 +1186,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) seek_video_frame_found = current_frame; // Are we close enough to decode the frame? and is this frame # valid? - if ((current_frame < (requested_frame - 20)) or (current_frame == -1)) - { + if ((current_frame < (requested_frame - 20)) or (current_frame == -1)) { // Remove frame and packet RemoveAVFrame(pFrame); @@ -866,10 +1208,10 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) AVFrame *my_frame = pFrame; // Add video frame to list of processing video frames - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_video_frames[current_frame] = current_frame; - #pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt) +#pragma omp task firstprivate(current_frame, my_frame, height, width, video_length, pix_fmt) { // Create variables for a RGB Frame (since most videos are not in RGB, we must convert it) AVFrame *pFrameRGB = NULL; @@ -886,14 +1228,14 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) // without losing quality. NOTE: We cannot go smaller than the timeline itself, or the add_layer timeline // method will scale it back to timeline size before scaling it smaller again. This needs to be fixed in // the future. - int max_width = Settings::Instance()->MAX_WIDTH; + int max_width = openshot::Settings::Instance()->MAX_WIDTH; if (max_width <= 0) max_width = info.width; - int max_height = Settings::Instance()->MAX_HEIGHT; + int max_height = openshot::Settings::Instance()->MAX_HEIGHT; if (max_height <= 0) max_height = info.height; - Clip* parent = (Clip*) GetClip(); + Clip *parent = (Clip *) GetClip(); if (parent) { if (parent->scale == SCALE_FIT || parent->scale == SCALE_STRETCH) { // Best fit or Stretch scaling (based on max timeline size * scaling keyframes) @@ -914,8 +1256,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) if (width_size.width() >= max_width && width_size.height() >= max_height) { max_width = max(max_width, width_size.width()); max_height = max(max_height, width_size.height()); - } - else { + } else { max_width = max(max_width, height_size.width()); max_height = max(max_height, height_size.height()); } @@ -949,7 +1290,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) // Determine required buffer size and allocate buffer numBytes = AV_GET_IMAGE_SIZE(PIX_FMT_RGBA, width, height); - #pragma omp critical (video_buffer) +#pragma omp critical (video_buffer) buffer = (uint8_t *) av_malloc(numBytes * sizeof(uint8_t)); // Copy picture data from one AVFrame (or AVPicture) to another one. @@ -976,7 +1317,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) working_cache.Add(f); // Keep track of last last_video_frame - #pragma omp critical (video_buffer) +#pragma omp critical (video_buffer) last_video_frame = f; // Free the RGB image @@ -989,7 +1330,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) // Remove video frame from list of processing video frames { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_video_frames.erase(current_frame); processed_video_frames[current_frame] = current_frame; } @@ -1002,15 +1343,13 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) } // Process an audio packet -void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample) -{ +void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_frame, int starting_sample) { // Track 1st audio packet after a successful seek if (!seek_audio_frame_found && is_seeking) seek_audio_frame_found = target_frame; // Are we close enough to decode the frame's audio? - if (target_frame < (requested_frame - 20)) - { + if (target_frame < (requested_frame - 20)) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessAudioPacket (Skipped)", "requested_frame", requested_frame, "target_frame", target_frame, "starting_sample", starting_sample, "", -1, "", -1, "", -1); @@ -1031,9 +1370,9 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // re-initialize buffer size (it gets changed in the avcodec_decode_audio2 method call) int buf_size = AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE; - #pragma omp critical (ProcessAudioPacket) +#pragma omp critical (ProcessAudioPacket) { - #if IS_FFMPEG_3_2 +#if IS_FFMPEG_3_2 int ret = 0; frame_finished = 1; while((packet->size > 0 || (!packet->data && frame_finished)) && ret >= 0) { @@ -1060,7 +1399,7 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr { ret = -1; } - #else +#else int used = avcodec_decode_audio4(aCodecCtx, audio_frame, &frame_finished, packet); #endif } @@ -1068,12 +1407,12 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr if (frame_finished) { // determine how many samples were decoded - int planar = av_sample_fmt_is_planar((AVSampleFormat)AV_GET_CODEC_PIXEL_FORMAT(aStream, aCodecCtx)); + int planar = av_sample_fmt_is_planar((AVSampleFormat) AV_GET_CODEC_PIXEL_FORMAT(aStream, aCodecCtx)); int plane_size = -1; data_size = av_samples_get_buffer_size(&plane_size, - AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels, - audio_frame->nb_samples, - (AVSampleFormat)(AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1); + AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels, + audio_frame->nb_samples, + (AVSampleFormat) (AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx)), 1); // Calculate total number of samples packet_samples = audio_frame->nb_samples * AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channels; @@ -1099,12 +1438,11 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // Add audio frame to list of processing audio frames { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_audio_frames.insert(pair(previous_packet_location.frame, previous_packet_location.frame)); } - while (pts_remaining_samples) - { + while (pts_remaining_samples) { // Get Samples per frame (for this frame number) int samples_per_frame = Frame::GetSamplesPerFrame(previous_packet_location.frame, info.fps, info.sample_rate, info.channels); @@ -1123,7 +1461,7 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // Add audio frame to list of processing audio frames { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_audio_frames.insert(pair(previous_packet_location.frame, previous_packet_location.frame)); } @@ -1150,24 +1488,24 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // setup resample context avr = SWR_ALLOC(); - av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0); + av_opt_set_int(avr, "in_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0); av_opt_set_int(avr, "out_channel_layout", AV_GET_CODEC_ATTRIBUTES(aStream, aCodecCtx)->channel_layout, 0); - av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0); - av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0); - av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0); - av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0); - av_opt_set_int(avr, "in_channels", info.channels, 0); - av_opt_set_int(avr, "out_channels", info.channels, 0); + av_opt_set_int(avr, "in_sample_fmt", AV_GET_SAMPLE_FORMAT(aStream, aCodecCtx), 0); + av_opt_set_int(avr, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0); + av_opt_set_int(avr, "in_sample_rate", info.sample_rate, 0); + av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0); + av_opt_set_int(avr, "in_channels", info.channels, 0); + av_opt_set_int(avr, "out_channels", info.channels, 0); int r = SWR_INIT(avr); // Convert audio samples - nb_samples = SWR_CONVERT(avr, // audio resample context - audio_converted->data, // output data pointers - audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown) - audio_converted->nb_samples, // maximum number of samples that the output buffer can hold - audio_frame->data, // input data pointers - audio_frame->linesize[0], // input plane size, in bytes (0 if unknown) - audio_frame->nb_samples); // number of input samples to convert + nb_samples = SWR_CONVERT(avr, // audio resample context + audio_converted->data, // output data pointers + audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown) + audio_converted->nb_samples, // maximum number of samples that the output buffer can hold + audio_frame->data, // input data pointers + audio_frame->linesize[0], // input plane size, in bytes (0 if unknown) + audio_frame->nb_samples); // number of input samples to convert // Copy audio samples over original samples memcpy(audio_buf, audio_converted->data[0], audio_converted->nb_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16) * info.channels); @@ -1183,8 +1521,7 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr int64_t starting_frame_number = -1; bool partial_frame = true; - for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) - { + for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) { // Array of floats (to hold samples for each channel) starting_frame_number = target_frame; int channel_buffer_size = packet_samples / info.channels; @@ -1198,11 +1535,9 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // Toggle through each channel number, since channel data is stored like (left right left right) int channel = 0; int position = 0; - for (int sample = 0; sample < packet_samples; sample++) - { + for (int sample = 0; sample < packet_samples; sample++) { // Only add samples for current channel - if (channel_filter == channel) - { + if (channel_filter == channel) { // Add sample (convert from (-32768 to 32768) to (-1.0 to 1.0)) channel_buffer[position] = audio_buf[sample] * (1.0f / (1 << 15)); @@ -1213,7 +1548,7 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // increment channel (if needed) if ((channel + 1) < info.channels) // move to next channel - channel ++; + channel++; else // reset channel channel = 0; @@ -1222,9 +1557,8 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // Loop through samples, and add them to the correct frames int start = starting_sample; int remaining_samples = channel_buffer_size; - float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer - while (remaining_samples > 0) - { + float *iterate_channel_buffer = channel_buffer; // pointer to channel buffer + while (remaining_samples > 0) { // Get Samples per frame (for this frame number) int samples_per_frame = Frame::GetSamplesPerFrame(starting_frame_number, info.fps, info.sample_rate, info.channels); @@ -1278,7 +1612,7 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr // Remove audio frame from list of processing audio frames { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); // Update all frames as completed for (int64_t f = target_frame; f < starting_frame_number; f++) { // Remove the frame # from the processing list. NOTE: If more than one thread is @@ -1307,10 +1641,8 @@ void FFmpegReader::ProcessAudioPacket(int64_t requested_frame, int64_t target_fr } - // Seek to a specific frame. This is not always frame accurate, it's more of an estimation on many codecs. -void FFmpegReader::Seek(int64_t requested_frame) -{ +void FFmpegReader::Seek(int64_t requested_frame) { // Adjust for a requested frame that is too small or too large if (requested_frame < 1) requested_frame = 1; @@ -1320,7 +1652,7 @@ void FFmpegReader::Seek(int64_t requested_frame) int processing_video_frames_size = 0; int processing_audio_frames_size = 0; { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_video_frames_size = processing_video_frames.size(); processing_audio_frames_size = processing_audio_frames.size(); } @@ -1331,7 +1663,7 @@ void FFmpegReader::Seek(int64_t requested_frame) // Wait for any processing frames to complete while (processing_video_frames_size + processing_audio_frames_size > 0) { usleep(2500); - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_video_frames_size = processing_video_frames.size(); processing_audio_frames_size = processing_audio_frames.size(); } @@ -1342,7 +1674,7 @@ void FFmpegReader::Seek(int64_t requested_frame) // Clear processed lists { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); processing_audio_frames.clear(); processing_video_frames.clear(); processed_video_frames.clear(); @@ -1369,8 +1701,7 @@ void FFmpegReader::Seek(int64_t requested_frame) // If seeking near frame 1, we need to close and re-open the file (this is more reliable than seeking) int buffer_amount = max(OPEN_MP_NUM_PROCESSORS, 8); - if (requested_frame - buffer_amount < 20) - { + if (requested_frame - buffer_amount < 20) { // Close and re-open file (basically seeking to frame 1) Close(); Open(); @@ -1388,21 +1719,18 @@ void FFmpegReader::Seek(int64_t requested_frame) } seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek seek_video_frame_found = 0; // used to detect which frames to throw away after a seek - } - else - { + + } else { // Seek to nearest key-frame (aka, i-frame) bool seek_worked = false; int64_t seek_target = 0; // Seek video stream (if any) - if (!seek_worked && info.has_video) - { + if (!seek_worked && info.has_video) { seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount); if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) { fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME); - } else - { + } else { // VIDEO SEEK is_video_seek = true; seek_worked = true; @@ -1410,13 +1738,11 @@ void FFmpegReader::Seek(int64_t requested_frame) } // Seek audio stream (if not already seeked... and if an audio stream is found) - if (!seek_worked && info.has_audio) - { + if (!seek_worked && info.has_audio) { seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount); if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) { fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME); - } else - { + } else { // AUDIO SEEK is_video_seek = false; seek_worked = true; @@ -1424,8 +1750,7 @@ void FFmpegReader::Seek(int64_t requested_frame) } // Was the seek successful? - if (seek_worked) - { + if (seek_worked) { // Flush audio buffer if (info.has_audio) avcodec_flush_buffers(aCodecCtx); @@ -1448,9 +1773,7 @@ void FFmpegReader::Seek(int64_t requested_frame) seek_audio_frame_found = 0; // used to detect which frames to throw away after a seek seek_video_frame_found = 0; // used to detect which frames to throw away after a seek - } - else - { + } else { // seek failed is_seeking = false; seeking_pts = 0; @@ -1472,10 +1795,9 @@ void FFmpegReader::Seek(int64_t requested_frame) } // Get the PTS for the current video packet -int64_t FFmpegReader::GetVideoPTS() -{ +int64_t FFmpegReader::GetVideoPTS() { int64_t current_pts = 0; - if(packet->dts != AV_NOPTS_VALUE) + if (packet->dts != AV_NOPTS_VALUE) current_pts = packet->dts; // Return adjusted PTS @@ -1483,11 +1805,9 @@ int64_t FFmpegReader::GetVideoPTS() } // Update PTS Offset (if any) -void FFmpegReader::UpdatePTSOffset(bool is_video) -{ +void FFmpegReader::UpdatePTSOffset(bool is_video) { // Determine the offset between the PTS and Frame number (only for 1st frame) - if (is_video) - { + if (is_video) { // VIDEO PACKET if (video_pts_offset == 99999) // Has the offset been set yet? { @@ -1497,9 +1817,7 @@ void FFmpegReader::UpdatePTSOffset(bool is_video) // debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::UpdatePTSOffset (Video)", "video_pts_offset", video_pts_offset, "is_video", is_video, "", -1, "", -1, "", -1, "", -1); } - } - else - { + } else { // AUDIO PACKET if (audio_pts_offset == 99999) // Has the offset been set yet? { @@ -1513,8 +1831,7 @@ void FFmpegReader::UpdatePTSOffset(bool is_video) } // Convert PTS into Frame Number -int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) -{ +int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) { // Apply PTS offset pts = pts + video_pts_offset; int64_t previous_video_frame = current_video_frame; @@ -1534,10 +1851,10 @@ int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) if (frame == previous_video_frame) { // return -1 frame number frame = -1; - } - else + } else { // Increment expected frame current_video_frame++; + } if (current_video_frame < frame) // has missing frames @@ -1545,7 +1862,7 @@ int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) // Sometimes frames are missing due to varying timestamps, or they were dropped. Determine // if we are missing a video frame. - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); while (current_video_frame < frame) { if (!missing_video_frames.count(current_video_frame)) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ConvertVideoPTStoFrame (tracking missing frame)", "current_video_frame", current_video_frame, "previous_video_frame", previous_video_frame, "", -1, "", -1, "", -1, "", -1); @@ -1566,8 +1883,7 @@ int64_t FFmpegReader::ConvertVideoPTStoFrame(int64_t pts) } // Convert Frame Number into Video PTS -int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) -{ +int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) { // Get timestamp of this frame (in seconds) double seconds = double(frame_number) / info.fps.ToDouble(); @@ -1579,8 +1895,7 @@ int64_t FFmpegReader::ConvertFrameToVideoPTS(int64_t frame_number) } // Convert Frame Number into Video PTS -int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) -{ +int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) { // Get timestamp of this frame (in seconds) double seconds = double(frame_number) / info.fps.ToDouble(); @@ -1592,8 +1907,7 @@ int64_t FFmpegReader::ConvertFrameToAudioPTS(int64_t frame_number) } // Calculate Starting video frame and sample # for an audio PTS -AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) -{ +AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) { // Apply PTS offset pts = pts + audio_pts_offset; @@ -1626,8 +1940,7 @@ AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) // Compare to previous audio packet (and fix small gaps due to varying PTS timestamps) if (previous_packet_location.frame != -1) { - if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) - { + if (location.is_near(previous_packet_location, samples_per_frame, samples_per_frame)) { int64_t orig_frame = location.frame; int orig_start = location.sample_start; @@ -1642,7 +1955,7 @@ AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (Audio Gap Ignored - too big)", "Previous location frame", previous_packet_location.frame, "Target Frame", location.frame, "Target Audio Sample", location.sample_start, "pts", pts, "", -1, "", -1); - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); for (int64_t audio_frame = previous_packet_location.frame; audio_frame < location.frame; audio_frame++) { if (!missing_audio_frames.count(audio_frame)) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAudioPTSLocation (tracking missing frame)", "missing_audio_frame", audio_frame, "previous_audio_frame", previous_packet_location.frame, "new location frame", location.frame, "", -1, "", -1, "", -1); @@ -1660,12 +1973,10 @@ AudioLocation FFmpegReader::GetAudioPTSLocation(int64_t pts) } // Create a new Frame (or return an existing one) and add it to the working queue. -std::shared_ptr FFmpegReader::CreateFrame(int64_t requested_frame) -{ +std::shared_ptr FFmpegReader::CreateFrame(int64_t requested_frame) { // Check working cache std::shared_ptr output = working_cache.GetFrame(requested_frame); - if (!output) - { + if (!output) { // Create a new frame on the working cache output = std::make_shared(requested_frame, info.width, info.height, "#000000", Frame::GetSamplesPerFrame(requested_frame, info.fps, info.sample_rate, info.channels), info.channels); output->SetPixelRatio(info.pixel_ratio.num, info.pixel_ratio.den); // update pixel ratio @@ -1689,20 +2000,21 @@ bool FFmpegReader::IsPartialFrame(int64_t requested_frame) { // Sometimes a seek gets partial frames, and we need to remove them bool seek_trash = false; int64_t max_seeked_frame = seek_audio_frame_found; // determine max seeked frame - if (seek_video_frame_found > max_seeked_frame) + if (seek_video_frame_found > max_seeked_frame) { max_seeked_frame = seek_video_frame_found; + } if ((info.has_audio && seek_audio_frame_found && max_seeked_frame >= requested_frame) || - (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) - seek_trash = true; + (info.has_video && seek_video_frame_found && max_seeked_frame >= requested_frame)) { + seek_trash = true; + } return seek_trash; } // Check if a frame is missing and attempt to replace it's frame image (and -bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) -{ +bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) { // Lock - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); // Init # of times this frame has been checked so far int checked_count = 0; @@ -1731,9 +2043,9 @@ bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) if (checked_count > 8 && !missing_video_frames.count(requested_frame) && !processing_audio_frames.count(requested_frame) && processed_audio_frames.count(requested_frame) && last_frame && last_video_frame->has_image_data && aCodecId == AV_CODEC_ID_MP3 && (vCodecId == AV_CODEC_ID_MJPEGB || vCodecId == AV_CODEC_ID_MJPEG)) { - missing_video_frames.insert(pair(requested_frame, last_video_frame->number)); - missing_video_frames_source.insert(pair(last_video_frame->number, requested_frame)); - missing_frames.Add(last_video_frame); + missing_video_frames.insert(pair(requested_frame, last_video_frame->number)); + missing_video_frames_source.insert(pair(last_video_frame->number, requested_frame)); + missing_frames.Add(last_video_frame); } } @@ -1798,17 +2110,15 @@ bool FFmpegReader::CheckMissingFrame(int64_t requested_frame) } // Check the working queue, and move finished frames to the finished queue -void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame) -{ +void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_frame) { // Loop through all working queue frames - bool checked_count_tripped = false; - int max_checked_count = 80; + bool checked_count_tripped = false; + int max_checked_count = 80; // Check if requested frame is 'missing' CheckMissingFrame(requested_frame); - while (true) - { + while (true) { // Get the front frame of working cache std::shared_ptr f(working_cache.GetSmallestFrame()); @@ -1832,17 +2142,17 @@ void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_fram bool is_video_ready = false; bool is_audio_ready = false; { // limit scope of next few lines - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); is_video_ready = processed_video_frames.count(f->number); is_audio_ready = processed_audio_frames.count(f->number); // Get check count for this frame checked_frames_size = checked_frames.size(); - if (!checked_count_tripped || f->number >= requested_frame) - checked_count = checked_frames[f->number]; - else - // Force checked count over the limit - checked_count = max_checked_count; + if (!checked_count_tripped || f->number >= requested_frame) + checked_count = checked_frames[f->number]; + else + // Force checked count over the limit + checked_count = max_checked_count; } if (previous_packet_location.frame == f->number && !end_of_stream) @@ -1858,8 +2168,8 @@ void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_fram // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (exceeded checked_count)", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size); - // Trigger checked count tripped mode (clear out all frames before requested frame) - checked_count_tripped = true; + // Trigger checked count tripped mode (clear out all frames before requested frame) + checked_count_tripped = true; if (info.has_video && !is_video_ready && last_video_frame) { // Copy image from last frame @@ -1877,13 +2187,11 @@ void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_fram ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames", "requested_frame", requested_frame, "frame_number", f->number, "is_video_ready", is_video_ready, "is_audio_ready", is_audio_ready, "checked_count", checked_count, "checked_frames_size", checked_frames_size); // Check if working frame is final - if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash) - { + if ((!end_of_stream && is_video_ready && is_audio_ready) || end_of_stream || is_seek_trash) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", "requested_frame", requested_frame, "f->number", f->number, "is_seek_trash", is_seek_trash, "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "end_of_stream", end_of_stream); - if (!is_seek_trash) - { + if (!is_seek_trash) { // Add missing image (if needed - sometimes end_of_stream causes frames with only audio) if (info.has_video && !is_video_ready && last_video_frame) // Copy image from last frame @@ -1897,15 +2205,15 @@ void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_fram // Add to missing cache (if another frame depends on it) { - const GenericScopedLock lock(processingCriticalSection); + const GenericScopedLock lock(processingCriticalSection); if (missing_video_frames_source.count(f->number)) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (add frame to missing cache)", "f->number", f->number, "is_seek_trash", is_seek_trash, "Missing Cache Count", missing_frames.Count(), "Working Cache Count", working_cache.Count(), "Final Cache Count", final_cache.Count(), "", -1); missing_frames.Add(f); } - // Remove from 'checked' count - checked_frames.erase(f->number); + // Remove from 'checked' count + checked_frames.erase(f->number); } // Remove frame from working cache @@ -1918,18 +2226,19 @@ void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_fram // Seek trash, so delete the frame from the working cache, and never add it to the final cache. working_cache.Remove(f->number); } - } - else + + } else { // Stop looping break; + } } } // Check for the correct frames per second (FPS) value by scanning the 1st few seconds of video packets. -void FFmpegReader::CheckFPS() -{ +void FFmpegReader::CheckFPS() { check_fps = true; + int first_second_counter = 0; int second_second_counter = 0; int third_second_counter = 0; @@ -1939,19 +2248,16 @@ void FFmpegReader::CheckFPS() int64_t pts = 0; // Loop through the stream - while (true) - { + while (true) { // Get the next packet (if any) if (GetNextPacket() < 0) // Break loop when no more packets found break; // Video packet - if (packet->stream_index == videoStream) - { + if (packet->stream_index == videoStream) { // Check if the AVFrame is finished and set it - if (GetAVFrame()) - { + if (GetAVFrame()) { // Update PTS / Frame Offset (if any) UpdatePTSOffset(true); @@ -2026,13 +2332,11 @@ void FFmpegReader::CheckFPS() } // Remove AVFrame from cache (and deallocate it's memory) -void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame) -{ - // Remove pFrame (if exists) - if (remove_frame) - { - // Free memory - #pragma omp critical (packet_cache) +void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) { + // Remove pFrame (if exists) + if (remove_frame) { + // Free memory +#pragma omp critical (packet_cache) { av_freep(&remove_frame->data[0]); #ifndef WIN32 @@ -2043,24 +2347,21 @@ void FFmpegReader::RemoveAVFrame(AVFrame* remove_frame) } // Remove AVPacket from cache (and deallocate it's memory) -void FFmpegReader::RemoveAVPacket(AVPacket* remove_packet) -{ +void FFmpegReader::RemoveAVPacket(AVPacket *remove_packet) { // deallocate memory for packet - AV_FREE_PACKET(remove_packet); + AV_FREE_PACKET(remove_packet); // Delete the object delete remove_packet; } /// Get the smallest video frame that is still being processed -int64_t FFmpegReader::GetSmallestVideoFrame() -{ +int64_t FFmpegReader::GetSmallestVideoFrame() { // Loop through frame numbers map::iterator itr; int64_t smallest_frame = -1; - const GenericScopedLock lock(processingCriticalSection); - for(itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr) - { + const GenericScopedLock lock(processingCriticalSection); + for (itr = processing_video_frames.begin(); itr != processing_video_frames.end(); ++itr) { if (itr->first < smallest_frame || smallest_frame == -1) smallest_frame = itr->first; } @@ -2070,14 +2371,12 @@ int64_t FFmpegReader::GetSmallestVideoFrame() } /// Get the smallest audio frame that is still being processed -int64_t FFmpegReader::GetSmallestAudioFrame() -{ +int64_t FFmpegReader::GetSmallestAudioFrame() { // Loop through frame numbers map::iterator itr; int64_t smallest_frame = -1; - const GenericScopedLock lock(processingCriticalSection); - for(itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr) - { + const GenericScopedLock lock(processingCriticalSection); + for (itr = processing_audio_frames.begin(); itr != processing_audio_frames.end(); ++itr) { if (itr->first < smallest_frame || smallest_frame == -1) smallest_frame = itr->first; } @@ -2111,18 +2410,16 @@ void FFmpegReader::SetJson(string value) { // Parse JSON string into JSON objects Json::Value root; Json::Reader reader; - bool success = reader.parse( value, root ); + bool success = reader.parse(value, root); if (!success) // Raise exception throw InvalidJSON("JSON could not be parsed (or is invalid)", ""); - try - { + try { // Set all values that match SetJsonValue(root); } - catch (exception e) - { + catch (exception e) { // Error parsing JSON (or missing keys) throw InvalidJSON("JSON is invalid (missing keys or invalid data types)", ""); } @@ -2139,8 +2436,7 @@ void FFmpegReader::SetJsonValue(Json::Value root) { path = root["path"].asString(); // Re-Open path, and re-init everything (if needed) - if (is_open) - { + if (is_open) { Close(); Open(); } diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 5d09341c..072ac6d7 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -32,14 +32,58 @@ using namespace openshot; +#if IS_FFMPEG_3_2 +#pragma message "You are compiling with experimental hardware encode" +#else +#pragma message "You are compiling only with software encode" +#endif + +#if IS_FFMPEG_3_2 +int hw_en_on = 1; // Is set in UI +int hw_en_supported = 0; // Is set by FFmpegWriter +AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE; +AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI; +static AVBufferRef *hw_device_ctx = NULL; +AVFrame *hw_frame = NULL; + +static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height) +{ + AVBufferRef *hw_frames_ref; + AVHWFramesContext *frames_ctx = NULL; + int err = 0; + + if (!(hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx))) { + fprintf(stderr, "Failed to create HW frame context.\n"); + return -1; + } + frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data); + frames_ctx->format = hw_en_av_pix_fmt; + frames_ctx->sw_format = AV_PIX_FMT_NV12; + frames_ctx->width = width; + frames_ctx->height = height; + frames_ctx->initial_pool_size = 20; + if ((err = av_hwframe_ctx_init(hw_frames_ref)) < 0) { + fprintf(stderr, "Failed to initialize HW frame context." + "Error code: %s\n",av_err2str(err)); + av_buffer_unref(&hw_frames_ref); + return err; + } + ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref); + if (!ctx->hw_frames_ctx) + err = AVERROR(ENOMEM); + + av_buffer_unref(&hw_frames_ref); + return err; +} +#endif + FFmpegWriter::FFmpegWriter(string path) : path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), audio_pts(0), video_pts(0), samples(NULL), audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0), initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(8), num_of_rescalers(32), rescaler_position(0), video_codec(NULL), audio_codec(NULL), is_writing(false), write_video_count(0), write_audio_count(0), original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false), - write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) -{ + write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) { // Disable audio & video (so they can be independently enabled) info.has_audio = false; @@ -53,8 +97,7 @@ FFmpegWriter::FFmpegWriter(string path) : } // Open the writer -void FFmpegWriter::Open() -{ +void FFmpegWriter::Open() { if (!is_open) { // Open the writer is_open = true; @@ -76,8 +119,7 @@ void FFmpegWriter::Open() } // auto detect format (from path) -void FFmpegWriter::auto_detect_format() -{ +void FFmpegWriter::auto_detect_format() { // Auto detect the output format from the name. default is mpeg. fmt = av_guess_format(NULL, path.c_str(), NULL); if (!fmt) @@ -102,8 +144,7 @@ void FFmpegWriter::auto_detect_format() } // initialize streams -void FFmpegWriter::initialize_streams() -{ +void FFmpegWriter::initialize_streams() { ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::initialize_streams", "fmt->video_codec", fmt->video_codec, "fmt->audio_codec", fmt->audio_codec, "AV_CODEC_ID_NONE", AV_CODEC_ID_NONE, "", -1, "", -1, "", -1); // Add the audio and video streams using the default format codecs and initialize the codecs @@ -119,12 +160,75 @@ void FFmpegWriter::initialize_streams() } // Set video export options -void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) -{ +void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) { // Set the video options - if (codec.length() > 0) - { - AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str()); + if (codec.length() > 0) { + AVCodec *new_codec; + // Check if the codec selected is a hardware accelerated codec +#if IS_FFMPEG_3_2 + #if defined(__linux__) + if ( (strcmp(codec.c_str(),"h264_vaapi") == 0)) { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 1; + hw_en_supported = 1; + hw_en_av_pix_fmt = AV_PIX_FMT_VAAPI; + hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI; + } + else { + if ( (strcmp(codec.c_str(),"h264_nvenc") == 0)) { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 1; + hw_en_supported = 1; + hw_en_av_pix_fmt = AV_PIX_FMT_CUDA; + hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA; + } + else { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 0; + hw_en_supported = 0; + } + } +#elif defined(_WIN32) + if ( (strcmp(codec.c_str(),"h264_dxva2") == 0)) { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 1; + hw_en_supported = 1; + hw_en_av_pix_fmt = AV_PIX_FMT_DXVA2_VLD; + hw_en_av_device_type = AV_HWDEVICE_TYPE_DXVA2; + } + else { + if ( (strcmp(codec.c_str(),"h264_nvenc") == 0)) { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 1; + hw_en_supported = 1; + hw_en_av_pix_fmt = AV_PIX_FMT_CUDA; + hw_en_av_device_type = AV_HWDEVICE_TYPE_CUDA; + } + else { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 0; + hw_en_supported = 0; + } + } +#elif defined(__APPLE__) + if ( (strcmp(codec.c_str(),"h264_videotoolbox") == 0)) { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 1; + hw_en_supported = 1; + hw_en_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX; + hw_en_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; + } + else { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = 0; + hw_en_supported = 0; + } +#else // is FFmpeg 3 but not linux + new_codec = avcodec_find_encoder_by_name(codec.c_str()); +#endif //__linux__ +#else // not ffmpeg 3 + new_codec = avcodec_find_encoder_by_name(codec.c_str()); +#endif //IS_FFMPEG_3_2 if (new_codec == NULL) throw InvalidCodec("A valid video codec could not be found for this file.", path); else { @@ -135,8 +239,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, i fmt->video_codec = new_codec->id; } } - if (fps.num > 0) - { + if (fps.num > 0) { // Set frames per second (if provided) info.fps.num = fps.num; info.fps.den = fps.den; @@ -149,14 +252,13 @@ void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, i info.width = width; if (height >= 1) info.height = height; - if (pixel_ratio.num > 0) - { + if (pixel_ratio.num > 0) { info.pixel_ratio.num = pixel_ratio.num; info.pixel_ratio.den = pixel_ratio.den; } - if (bit_rate >= 1000) // bit_rate is the bitrate in b/s + if (bit_rate >= 1000) // bit_rate is the bitrate in b/s info.video_bit_rate = bit_rate; - if ((bit_rate >= 0) && (bit_rate < 64) ) // bit_rate is the bitrate in crf + if ((bit_rate >= 0) && (bit_rate < 64)) // bit_rate is the bitrate in crf info.video_bit_rate = bit_rate; info.interlaced_frame = interlaced; @@ -179,16 +281,13 @@ void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, i } // Set audio export options -void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate) -{ +void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, ChannelLayout channel_layout, int bit_rate) { // Set audio options - if (codec.length() > 0) - { + if (codec.length() > 0) { AVCodec *new_codec = avcodec_find_encoder_by_name(codec.c_str()); if (new_codec == NULL) throw InvalidCodec("A valid audio codec could not be found for this file.", path); - else - { + else { // Set audio codec info.acodec = new_codec->name; @@ -217,8 +316,7 @@ void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate } // Set custom options (some codecs accept additional params) -void FFmpegWriter::SetOption(StreamType stream, string name, string value) -{ +void FFmpegWriter::SetOption(StreamType stream, string name, string value) { // Declare codec context AVCodecContext *c = NULL; AVStream *st = NULL; @@ -228,13 +326,11 @@ void FFmpegWriter::SetOption(StreamType stream, string name, string value) st = video_st; // Get codec context c = AV_GET_CODEC_PAR_CONTEXT(st, video_codec); - } - else if (info.has_audio && stream == AUDIO_STREAM && audio_st) { + } else if (info.has_audio && stream == AUDIO_STREAM && audio_st) { st = audio_st; // Get codec context c = AV_GET_CODEC_PAR_CONTEXT(st, audio_codec); - } - else + } else throw NoStreamsFound("The stream was not found. Be sure to call PrepareStreams() first.", path); // Init AVOption @@ -247,9 +343,8 @@ void FFmpegWriter::SetOption(StreamType stream, string name, string value) // Was option found? if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" || - name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" || - name == "crf")) - { + name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" || + name == "crf")) { // Check for specific named options if (name == "g") // Set gop_size @@ -299,65 +394,79 @@ void FFmpegWriter::SetOption(StreamType stream, string name, string value) // encode quality and special settings like lossless // This might be better in an extra methods as more options // and way to set quality are possible - #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 39, 101) - switch (c->codec_id) { - #if (LIBAVCODEC_VERSION_MAJOR >= 58) - case AV_CODEC_ID_AV1 : - c->bit_rate = 0; - av_opt_set_int(c->priv_data, "crf", min(stoi(value),63), 0); - break; - #endif - case AV_CODEC_ID_VP8 : - c->bit_rate = 10000000; - av_opt_set_int(c->priv_data, "crf", max(min(stoi(value),63),4), 0); // 4-63 - break; - case AV_CODEC_ID_VP9 : - c->bit_rate = 0; // Must be zero! - av_opt_set_int(c->priv_data, "crf", min(stoi(value),63), 0); // 0-63 - if (stoi(value) == 0) { - av_opt_set(c->priv_data, "preset", "veryslow", 0); - av_opt_set_int(c->priv_data, "lossless", 1, 0); - } - break; - case AV_CODEC_ID_H264 : - av_opt_set_int(c->priv_data, "crf", min(stoi(value),51), 0); // 0-51 - if (stoi(value) == 0) { - av_opt_set(c->priv_data, "preset", "veryslow", 0); - } - break; - case AV_CODEC_ID_H265 : - av_opt_set_int(c->priv_data, "crf", min(stoi(value),51), 0); // 0-51 - if (stoi(value) == 0) { - av_opt_set(c->priv_data, "preset", "veryslow", 0); - av_opt_set_int(c->priv_data, "lossless", 1, 0); - } - break; - default: - // If this codec doesn't support crf calculate a bitrate - // TODO: find better formula - double mbs = 15000000.0; - if (info.video_bit_rate > 0) { - if (info.video_bit_rate > 42) { - mbs = 380.0; - } - else { - mbs *= pow(0.912,info.video_bit_rate); - } - } - c->bit_rate = (int)(mbs); +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 39, 101) +#if IS_FFMPEG_3_2 + if (hw_en_on) { + double mbs = 15000000.0; + if (info.video_bit_rate > 0) { + if (info.video_bit_rate > 42) { + mbs = 380000.0; } - #endif - } - - else + else { + mbs *= pow(0.912,info.video_bit_rate); + } + } + c->bit_rate = (int)(mbs); + } else +#endif + { + switch (c->codec_id) { +#if (LIBAVCODEC_VERSION_MAJOR >= 58) + case AV_CODEC_ID_AV1 : + c->bit_rate = 0; + av_opt_set_int(c->priv_data, "crf", min(stoi(value),63), 0); + break; +#endif + case AV_CODEC_ID_VP8 : + c->bit_rate = 10000000; + av_opt_set_int(c->priv_data, "crf", max(min(stoi(value), 63), 4), 0); // 4-63 + break; + case AV_CODEC_ID_VP9 : + c->bit_rate = 0; // Must be zero! + av_opt_set_int(c->priv_data, "crf", min(stoi(value), 63), 0); // 0-63 + if (stoi(value) == 0) { + av_opt_set(c->priv_data, "preset", "veryslow", 0); + av_opt_set_int(c->priv_data, "lossless", 1, 0); + } + break; + case AV_CODEC_ID_H264 : + av_opt_set_int(c->priv_data, "crf", min(stoi(value), 51), 0); // 0-51 + if (stoi(value) == 0) { + av_opt_set(c->priv_data, "preset", "veryslow", 0); + } + break; + case AV_CODEC_ID_H265 : + av_opt_set_int(c->priv_data, "crf", min(stoi(value), 51), 0); // 0-51 + if (stoi(value) == 0) { + av_opt_set(c->priv_data, "preset", "veryslow", 0); + av_opt_set_int(c->priv_data, "lossless", 1, 0); + } + break; + default: + // If this codec doesn't support crf calculate a bitrate + // TODO: find better formula + double mbs = 15000000.0; + if (info.video_bit_rate > 0) { + if (info.video_bit_rate > 42) { + mbs = 380000.0; + } else { + mbs *= pow(0.912, info.video_bit_rate); + } + } + c->bit_rate = (int) (mbs); + } + } +#endif + } else { // Set AVOption AV_OPTION_SET(st, c->priv_data, name.c_str(), value.c_str(), c); + } ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::SetOption (" + (string)name + ")", "stream == VIDEO_STREAM", stream == VIDEO_STREAM, "", -1, "", -1, "", -1, "", -1, "", -1); - } - else + } else { throw InvalidOptions("The option is not valid for this codec.", path); + } } @@ -374,8 +483,7 @@ bool FFmpegWriter::IsValidCodec(string codec_name) { } // Prepare & initialize streams and open codecs -void FFmpegWriter::PrepareStreams() -{ +void FFmpegWriter::PrepareStreams() { if (!info.has_audio && !info.has_video) throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path); @@ -389,8 +497,7 @@ void FFmpegWriter::PrepareStreams() } // Write the file header (after the options are set) -void FFmpegWriter::WriteHeader() -{ +void FFmpegWriter::WriteHeader() { if (!info.has_audio && !info.has_video) throw InvalidOptions("No video or audio options have been set. You must set has_video or has_audio (or both).", path); @@ -400,20 +507,19 @@ void FFmpegWriter::WriteHeader() throw InvalidFile("Could not open or write file.", path); } - // Force the output filename (which doesn't always happen for some reason) - snprintf(oc->AV_FILENAME, sizeof(oc->AV_FILENAME), "%s", path.c_str()); + // Force the output filename (which doesn't always happen for some reason) + snprintf(oc->AV_FILENAME, sizeof(oc->AV_FILENAME), "%s", path.c_str()); // Write the stream header, if any // TODO: add avoptions / parameters instead of NULL // Add general metadata (if any) - for(std::map::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) - { + for (std::map::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) { av_dict_set(&oc->metadata, iter->first.c_str(), iter->second.c_str(), 0); } if (avformat_write_header(oc, NULL) != 0) { - throw InvalidFile("Could not write header to file.", path); + throw InvalidFile("Could not write header to file.", path); }; // Mark as 'written' @@ -423,8 +529,7 @@ void FFmpegWriter::WriteHeader() } // Add a frame to the queue waiting to be encoded. -void FFmpegWriter::WriteFrame(std::shared_ptr frame) -{ +void FFmpegWriter::WriteFrame(std::shared_ptr frame) { // Check for open reader (or throw exception) if (!is_open) throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path); @@ -440,15 +545,13 @@ void FFmpegWriter::WriteFrame(std::shared_ptr frame) ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame", "frame->number", frame->number, "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "cache_size", cache_size, "is_writing", is_writing, "", -1); // Write the frames once it reaches the correct cache size - if (spooled_video_frames.size() == cache_size || spooled_audio_frames.size() == cache_size) - { + if (spooled_video_frames.size() == cache_size || spooled_audio_frames.size() == cache_size) { // Is writer currently writing? if (!is_writing) // Write frames to video file write_queued_frames(); - else - { + else { // Write frames to video file write_queued_frames(); } @@ -459,8 +562,7 @@ void FFmpegWriter::WriteFrame(std::shared_ptr frame) } // Write all frames in the queue to the video file. -void FFmpegWriter::write_queued_frames() -{ +void FFmpegWriter::write_queued_frames() { ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_queued_frames", "spooled_video_frames.size()", spooled_video_frames.size(), "spooled_audio_frames.size()", spooled_audio_frames.size(), "", -1, "", -1, "", -1, "", -1); // Flip writing flag @@ -482,17 +584,16 @@ void FFmpegWriter::write_queued_frames() // Create blank exception bool has_error_encoding_video = false; - #pragma omp parallel +#pragma omp parallel { - #pragma omp single +#pragma omp single { // Process all audio frames (in a separate thread) if (info.has_audio && audio_st && !queued_audio_frames.empty()) write_audio_packets(false); // Loop through each queued image frame - while (!queued_video_frames.empty()) - { + while (!queued_video_frames.empty()) { // Get front frame (from the queue) std::shared_ptr frame = queued_video_frames.front(); @@ -509,22 +610,19 @@ void FFmpegWriter::write_queued_frames() } // end while } // end omp single - #pragma omp single +#pragma omp single { // Loop back through the frames (in order), and write them to the video file - while (!processed_frames.empty()) - { + while (!processed_frames.empty()) { // Get front frame (from the queue) std::shared_ptr frame = processed_frames.front(); - if (info.has_video && video_st) - { + if (info.has_video && video_st) { // Add to deallocate queue (so we can remove the AVFrames when we are done) deallocate_frames.push_back(frame); // Does this frame's AVFrame still exist - if (av_frames.count(frame)) - { + if (av_frames.count(frame)) { // Get AVFrame AVFrame *frame_final = av_frames[frame]; @@ -540,14 +638,12 @@ void FFmpegWriter::write_queued_frames() } // Loop through, and deallocate AVFrames - while (!deallocate_frames.empty()) - { + while (!deallocate_frames.empty()) { // Get front frame (from the queue) std::shared_ptr frame = deallocate_frames.front(); // Does this frame's AVFrame still exist - if (av_frames.count(frame)) - { + if (av_frames.count(frame)) { // Get AVFrame AVFrame *av_frame = av_frames[frame]; @@ -565,6 +661,7 @@ void FFmpegWriter::write_queued_frames() is_writing = false; } // end omp single + } // end omp parallel // Raise exception from main thread @@ -573,13 +670,11 @@ void FFmpegWriter::write_queued_frames() } // Write a block of frames from a reader -void FFmpegWriter::WriteFrame(ReaderBase* reader, int64_t start, int64_t length) -{ +void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::WriteFrame (from Reader)", "start", start, "length", length, "", -1, "", -1, "", -1, "", -1); // Loop through each frame (and encoded it) - for (int64_t number = start; number <= length; number++) - { + for (int64_t number = start; number <= length; number++) { // Get the frame std::shared_ptr f = reader->GetFrame(number); @@ -589,8 +684,7 @@ void FFmpegWriter::WriteFrame(ReaderBase* reader, int64_t start, int64_t length) } // Write the file trailer (after all frames are written) -void FFmpegWriter::WriteTrailer() -{ +void FFmpegWriter::WriteTrailer() { // Write any remaining queued frames to video file write_queued_frames(); @@ -614,8 +708,7 @@ void FFmpegWriter::WriteTrailer() } // Flush encoders -void FFmpegWriter::flush_encoders() -{ +void FFmpegWriter::flush_encoders() { if (info.has_audio && audio_codec && AV_GET_CODEC_TYPE(audio_st) == AVMEDIA_TYPE_AUDIO && AV_GET_CODEC_ATTRIBUTES(audio_st, audio_codec)->frame_size <= 1) return; #if (LIBAVFORMAT_VERSION_MAJOR < 58) @@ -623,15 +716,15 @@ void FFmpegWriter::flush_encoders() return; #endif - int error_code = 0; - int stop_encoding = 1; + int error_code = 0; + int stop_encoding = 1; - // FLUSH VIDEO ENCODER - if (info.has_video) + // FLUSH VIDEO ENCODER + if (info.has_video) for (;;) { // Increment PTS (in frames and scaled to the codec's timebase) - write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base); + write_video_count += av_rescale_q(1, (AVRational) {info.fps.den, info.fps.num}, video_codec->time_base); AVPacket pkt; av_init_packet(&pkt); @@ -645,7 +738,7 @@ void FFmpegWriter::flush_encoders() int got_packet = 0; int error_code = 0; - #if IS_FFMPEG_3_2 +#if IS_FFMPEG_3_2 #pragma omp critical (write_video_packet) { // Encode video packet (latest version of FFmpeg) @@ -669,34 +762,35 @@ void FFmpegWriter::flush_encoders() error_code = av_interleaved_write_frame(oc, &pkt); } } - #else +#else // IS_FFMPEG_3_2 - #if LIBAVFORMAT_VERSION_MAJOR >= 54 - // Encode video packet (older than FFmpeg 3.2) - error_code = avcodec_encode_video2(video_codec, &pkt, NULL, &got_packet); +#if LIBAVFORMAT_VERSION_MAJOR >= 54 + // Encode video packet (older than FFmpeg 3.2) + error_code = avcodec_encode_video2(video_codec, &pkt, NULL, &got_packet); - #else - // Encode video packet (even older version of FFmpeg) - int video_outbuf_size = 0; +#else + // Encode video packet (even older version of FFmpeg) + int video_outbuf_size = 0; - /* encode the image */ - int out_size = avcodec_encode_video(video_codec, NULL, video_outbuf_size, NULL); + /* encode the image */ + int out_size = avcodec_encode_video(video_codec, NULL, video_outbuf_size, NULL); - /* if zero size, it means the image was buffered */ - if (out_size > 0) { - if(video_codec->coded_frame->key_frame) - pkt.flags |= AV_PKT_FLAG_KEY; - pkt.data= video_outbuf; - pkt.size= out_size; + /* if zero size, it means the image was buffered */ + if (out_size > 0) { + if(video_codec->coded_frame->key_frame) + pkt.flags |= AV_PKT_FLAG_KEY; + pkt.data= video_outbuf; + pkt.size= out_size; - // got data back (so encode this frame) - got_packet = 1; - } - #endif - #endif + // got data back (so encode this frame) + got_packet = 1; + } +#endif // LIBAVFORMAT_VERSION_MAJOR >= 54 +#endif // IS_FFMPEG_3_2 if (error_code < 0) { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); + ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string) av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, + "", -1, "", -1, "", -1); } if (!got_packet) { stop_encoding = 1; @@ -726,8 +820,8 @@ void FFmpegWriter::flush_encoders() av_freep(&video_outbuf); } - // FLUSH AUDIO ENCODER - if (info.has_audio) + // FLUSH AUDIO ENCODER + if (info.has_audio) for (;;) { // Increment PTS (in samples and scaled to the codec's timebase) @@ -746,12 +840,12 @@ void FFmpegWriter::flush_encoders() /* encode the image */ int got_packet = 0; - #if IS_FFMPEG_3_2 +#if IS_FFMPEG_3_2 avcodec_send_frame(audio_codec, NULL); got_packet = 0; - #else +#else error_code = avcodec_encode_audio2(audio_codec, &pkt, NULL, &got_packet); - #endif +#endif if (error_code < 0) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::flush_encoders ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); } @@ -790,15 +884,23 @@ void FFmpegWriter::flush_encoders() } // Close the video codec -void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st) -{ +void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st) { AV_FREE_CONTEXT(video_codec); video_codec = NULL; +#if IS_FFMPEG_3_2 + // #if defined(__linux__) + if (hw_en_on && hw_en_supported) { + if (hw_device_ctx) { + av_buffer_unref(&hw_device_ctx); + hw_device_ctx = NULL; + } + } + // #endif +#endif } // Close the audio codec -void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st) -{ +void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st) { AV_FREE_CONTEXT(audio_codec); audio_codec = NULL; @@ -825,8 +927,7 @@ void FFmpegWriter::close_audio(AVFormatContext *oc, AVStream *st) } // Close the writer -void FFmpegWriter::Close() -{ +void FFmpegWriter::Close() { // Write trailer (if needed) if (!write_trailer) WriteTrailer(); @@ -869,24 +970,19 @@ void FFmpegWriter::Close() } // Add an AVFrame to the cache -void FFmpegWriter::add_avframe(std::shared_ptr frame, AVFrame* av_frame) -{ +void FFmpegWriter::add_avframe(std::shared_ptr frame, AVFrame *av_frame) { // Add AVFrame to map (if it does not already exist) - if (!av_frames.count(frame)) - { + if (!av_frames.count(frame)) { // Add av_frame av_frames[frame] = av_frame; - } - else - { + } else { // Do not add, and deallocate this AVFrame AV_FREE_FRAME(&av_frame); } } // Add an audio output stream -AVStream* FFmpegWriter::add_audio_stream() -{ +AVStream *FFmpegWriter::add_audio_stream() { AVCodecContext *c; AVStream *st; @@ -912,15 +1008,14 @@ AVStream* FFmpegWriter::add_audio_stream() // Set valid sample rate (or throw error) if (codec->supported_samplerates) { int i; - for (i = 0; codec->supported_samplerates[i] != 0; i++) - if (info.sample_rate == codec->supported_samplerates[i]) - { - // Set the valid sample rate - c->sample_rate = info.sample_rate; - break; - } - if (codec->supported_samplerates[i] == 0) - throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path); + for (i = 0; codec->supported_samplerates[i] != 0; i++) + if (info.sample_rate == codec->supported_samplerates[i]) { + // Set the valid sample rate + c->sample_rate = info.sample_rate; + break; + } + if (codec->supported_samplerates[i] == 0) + throw InvalidSampleRate("An invalid sample rate was detected for this codec.", path); } else // Set sample rate c->sample_rate = info.sample_rate; @@ -931,8 +1026,7 @@ AVStream* FFmpegWriter::add_audio_stream() if (codec->channel_layouts) { int i; for (i = 0; codec->channel_layouts[i] != 0; i++) - if (channel_layout == codec->channel_layouts[i]) - { + if (channel_layout == codec->channel_layouts[i]) { // Set valid channel layout c->channel_layout = channel_layout; break; @@ -945,8 +1039,7 @@ AVStream* FFmpegWriter::add_audio_stream() // Choose a valid sample_fmt if (codec->sample_fmts) { - for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) - { + for (int i = 0; codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++) { // Set sample format to 1st valid format (and then exit loop) c->sample_fmt = codec->sample_fmts[i]; break; @@ -972,8 +1065,7 @@ AVStream* FFmpegWriter::add_audio_stream() } // Add a video output stream -AVStream* FFmpegWriter::add_video_stream() -{ +AVStream *FFmpegWriter::add_video_stream() { AVCodecContext *c; AVStream *st; @@ -1001,13 +1093,34 @@ AVStream* FFmpegWriter::add_video_stream() } // Here should be the setting for low fixed bitrate // Defaults are used because mpeg2 otherwise had problems - } - else { - c->qmin = 0; - c->qmax = 63; + } else { + // Check if codec supports crf + switch (c->codec_id) { +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 39, 101) +#if (LIBAVCODEC_VERSION_MAJOR >= 58) + case AV_CODEC_ID_AV1 : +#endif + case AV_CODEC_ID_VP9 : + case AV_CODEC_ID_H265 : +#endif + case AV_CODEC_ID_VP8 : + case AV_CODEC_ID_H264 : + if (info.video_bit_rate < 40) { + c->qmin = 0; + c->qmax = 63; + } else { + c->qmin = info.video_bit_rate - 5; + c->qmax = 63; + } + break; + default: + // Here should be the setting for codecs that don't support crf + // For now defaults are used + break; + } } - //TODO: Implement variable bitrate feature (which actually works). This implementation throws +//TODO: Implement variable bitrate feature (which actually works). This implementation throws //invalid bitrate errors and rc buffer underflow errors, etc... //c->rc_min_rate = info.video_bit_rate; //c->rc_max_rate = info.video_bit_rate; @@ -1026,9 +1139,9 @@ AVStream* FFmpegWriter::add_video_stream() identically 1. */ c->time_base.num = info.video_timebase.num; c->time_base.den = info.video_timebase.den; - #if LIBAVFORMAT_VERSION_MAJOR >= 56 +#if LIBAVFORMAT_VERSION_MAJOR >= 56 c->framerate = av_inv_q(c->time_base); - #endif +#endif st->avg_frame_rate = av_inv_q(c->time_base); st->time_base.num = info.video_timebase.num; st->time_base.den = info.video_timebase.den; @@ -1052,31 +1165,31 @@ AVStream* FFmpegWriter::add_video_stream() #endif // Find all supported pixel formats for this codec - const PixelFormat* supported_pixel_formats = codec->pix_fmts; - while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) { - // Assign the 1st valid pixel format (if one is missing) - if (c->pix_fmt == PIX_FMT_NONE) - c->pix_fmt = *supported_pixel_formats; - ++supported_pixel_formats; - } + const PixelFormat *supported_pixel_formats = codec->pix_fmts; + while (supported_pixel_formats != NULL && *supported_pixel_formats != PIX_FMT_NONE) { + // Assign the 1st valid pixel format (if one is missing) + if (c->pix_fmt == PIX_FMT_NONE) + c->pix_fmt = *supported_pixel_formats; + ++supported_pixel_formats; + } - // Codec doesn't have any pix formats? - if (c->pix_fmt == PIX_FMT_NONE) { - if(fmt->video_codec == AV_CODEC_ID_RAWVIDEO) { - // Raw video should use RGB24 - c->pix_fmt = PIX_FMT_RGB24; + // Codec doesn't have any pix formats? + if (c->pix_fmt == PIX_FMT_NONE) { + if (fmt->video_codec == AV_CODEC_ID_RAWVIDEO) { + // Raw video should use RGB24 + c->pix_fmt = PIX_FMT_RGB24; #if (LIBAVFORMAT_VERSION_MAJOR < 58) - if (strcmp(fmt->name, "gif") != 0) - // If not GIF format, skip the encoding process - // Set raw picture flag (so we don't encode this video) - oc->oformat->flags |= AVFMT_RAWPICTURE; + if (strcmp(fmt->name, "gif") != 0) + // If not GIF format, skip the encoding process + // Set raw picture flag (so we don't encode this video) + oc->oformat->flags |= AVFMT_RAWPICTURE; #endif - } else { - // Set the default codec - c->pix_fmt = PIX_FMT_YUV420P; - } - } + } else { + // Set the default codec + c->pix_fmt = PIX_FMT_YUV420P; + } + } AV_COPY_PARAMS_FROM_CONTEXT(st, c); #if (LIBAVFORMAT_VERSION_MAJOR < 58) @@ -1089,8 +1202,7 @@ AVStream* FFmpegWriter::add_video_stream() } // open audio codec -void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) -{ +void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) { AVCodec *codec; AV_GET_CODEC_FROM_STREAM(st, audio_codec) @@ -1124,14 +1236,14 @@ void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) int s = AV_FIND_DECODER_CODEC_ID(st); switch (s) { - case AV_CODEC_ID_PCM_S16LE: - case AV_CODEC_ID_PCM_S16BE: - case AV_CODEC_ID_PCM_U16LE: - case AV_CODEC_ID_PCM_U16BE: - audio_input_frame_size >>= 1; - break; - default: - break; + case AV_CODEC_ID_PCM_S16LE: + case AV_CODEC_ID_PCM_S16BE: + case AV_CODEC_ID_PCM_U16LE: + case AV_CODEC_ID_PCM_U16BE: + audio_input_frame_size >>= 1; + break; + default: + break; } } else { // Set frame size based on the codec @@ -1153,24 +1265,66 @@ void FFmpegWriter::open_audio(AVFormatContext *oc, AVStream *st) audio_encoder_buffer = new uint8_t[audio_encoder_buffer_size]; // Add audio metadata (if any) - for(std::map::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) - { + for (std::map::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) { av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0); } ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_audio", "audio_codec->thread_count", audio_codec->thread_count, "audio_input_frame_size", audio_input_frame_size, "buffer_size", AVCODEC_MAX_AUDIO_FRAME_SIZE + MY_INPUT_BUFFER_PADDING_SIZE, "", -1, "", -1, "", -1); - } // open video codec -void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) -{ +void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { AVCodec *codec; AV_GET_CODEC_FROM_STREAM(st, video_codec) // Set number of threads equal to number of processors (not to exceed 16) video_codec->thread_count = min(FF_NUM_PROCESSORS, 16); +#if IS_FFMPEG_3_2 + if (hw_en_on && hw_en_supported) { + //char *dev_hw = NULL; + char adapter[256]; + char *adapter_ptr = NULL; + int adapter_num; + // Use the hw device given in the environment variable HW_EN_DEVICE_SET or the default if not set + adapter_num = openshot::Settings::Instance()->HW_EN_DEVICE_SET; + fprintf(stderr, "\n\nEncodiing Device Nr: %d\n", adapter_num); + if (adapter_num < 3 && adapter_num >=0) { +#if defined(__linux__) + snprintf(adapter,sizeof(adapter),"/dev/dri/renderD%d", adapter_num+128); + // Maybe 127 is better because the first card would be 1?! + adapter_ptr = adapter; +#elif defined(_WIN32) + adapter_ptr = NULL; +#elif defined(__APPLE__) + adapter_ptr = NULL; +#endif + } + else { + adapter_ptr = NULL; // Just to be sure + } +// Check if it is there and writable +#if defined(__linux__) + if( adapter_ptr != NULL && access( adapter_ptr, W_OK ) == 0 ) { +#elif defined(_WIN32) + if( adapter_ptr != NULL ) { +#elif defined(__APPLE__) + if( adapter_ptr != NULL ) { +#endif + ZmqLogger::Instance()->AppendDebugMethod("Encode Device present using device", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } + else { + adapter_ptr = NULL; // use default + ZmqLogger::Instance()->AppendDebugMethod("Encode Device not present using default", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); + } + if (av_hwdevice_ctx_create(&hw_device_ctx, hw_en_av_device_type, + adapter_ptr, NULL, 0) < 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::open_video : Codec name: ", info.vcodec.c_str(), -1, " ERROR creating\n", -1, "", -1, "", -1, "", -1, "", -1); + throw InvalidCodec("Could not create hwdevice", path); + } + } +#endif + /* find the video encoder */ codec = avcodec_find_encoder_by_name(info.vcodec.c_str()); if (!codec) @@ -1178,14 +1332,30 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) if (!codec) throw InvalidCodec("Could not find codec", path); - /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */ - if(video_codec->max_b_frames && video_codec->codec_id != AV_CODEC_ID_MPEG4 && video_codec->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec->codec_id != AV_CODEC_ID_MPEG2VIDEO) - video_codec->max_b_frames = 0; + /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */ + if (video_codec->max_b_frames && video_codec->codec_id != AV_CODEC_ID_MPEG4 && video_codec->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec->codec_id != AV_CODEC_ID_MPEG2VIDEO) + video_codec->max_b_frames = 0; // Init options AVDictionary *opts = NULL; av_dict_set(&opts, "strict", "experimental", 0); +#if IS_FFMPEG_3_2 + if (hw_en_on && hw_en_supported) { + video_codec->max_b_frames = 0; // At least this GPU doesn't support b-frames + video_codec->pix_fmt = hw_en_av_pix_fmt; + video_codec->profile = FF_PROFILE_H264_BASELINE | FF_PROFILE_H264_CONSTRAINED; + av_opt_set(video_codec->priv_data,"preset","slow",0); + av_opt_set(video_codec->priv_data,"tune","zerolatency",0); + av_opt_set(video_codec->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN); + // set hw_frames_ctx for encoder's AVCodecContext + int err; + if ((err = set_hwframe_ctx(video_codec, hw_device_ctx, info.width, info.height)) < 0) { + fprintf(stderr, "Failed to set hwframe context.\n"); + } + } +#endif + /* open the codec */ if (avcodec_open2(video_codec, codec, &opts) < 0) throw InvalidCodec("Could not open codec", path); @@ -1195,8 +1365,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) av_dict_free(&opts); // Add video metadata (if any) - for(std::map::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) - { + for (std::map::iterator iter = info.metadata.begin(); iter != info.metadata.end(); ++iter) { av_dict_set(&st->metadata, iter->first.c_str(), iter->second.c_str(), 0); } @@ -1205,9 +1374,8 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) } // write all queued frames' audio to the video file -void FFmpegWriter::write_audio_packets(bool final) -{ - #pragma omp task firstprivate(final) +void FFmpegWriter::write_audio_packets(bool is_final) { +#pragma omp task firstprivate(is_final) { // Init audio buffers / variables int total_frame_samples = 0; @@ -1218,14 +1386,13 @@ void FFmpegWriter::write_audio_packets(bool final) ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout // Create a new array (to hold all S16 audio samples, for the current queued frames - int16_t* all_queued_samples = (int16_t*)av_malloc((sizeof(int16_t)*(queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE))); - int16_t* all_resampled_samples = NULL; - int16_t* final_samples_planar = NULL; - int16_t* final_samples = NULL; + int16_t *all_queued_samples = (int16_t *) av_malloc((sizeof(int16_t) * (queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE))); + int16_t *all_resampled_samples = NULL; + int16_t *final_samples_planar = NULL; + int16_t *final_samples = NULL; // Loop through each queued audio frame - while (!queued_audio_frames.empty()) - { + while (!queued_audio_frames.empty()) { // Get front frame (from the queue) std::shared_ptr frame = queued_audio_frames.front(); @@ -1237,7 +1404,7 @@ void FFmpegWriter::write_audio_packets(bool final) // Get audio sample array - float* frame_samples_float = NULL; + float *frame_samples_float = NULL; // Get samples interleaved together (c1 c2 c1 c2 c1 c2) frame_samples_float = frame->GetInterleavedAudioSamples(sample_rate_in_frame, NULL, &samples_in_frame); @@ -1266,13 +1433,13 @@ void FFmpegWriter::write_audio_packets(bool final) int samples_position = 0; - ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "final", final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO); + ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets", "is_final", is_final, "total_frame_samples", total_frame_samples, "channel_layout_in_frame", channel_layout_in_frame, "channels_in_frame", channels_in_frame, "samples_in_frame", samples_in_frame, "LAYOUT_MONO", LAYOUT_MONO); // Keep track of the original sample format AVSampleFormat output_sample_fmt = audio_codec->sample_fmt; AVFrame *audio_frame = NULL; - if (!final) { + if (!is_final) { // Create input frame (and allocate arrays) audio_frame = AV_ALLOCATE_FRAME(); AV_RESET_FRAME(audio_frame); @@ -1280,28 +1447,23 @@ void FFmpegWriter::write_audio_packets(bool final) // Fill input frame with sample data avcodec_fill_audio_frame(audio_frame, channels_in_frame, AV_SAMPLE_FMT_S16, (uint8_t *) all_queued_samples, - audio_encoder_buffer_size, 0); + audio_encoder_buffer_size, 0); // Do not convert audio to planar format (yet). We need to keep everything interleaved at this point. - switch (audio_codec->sample_fmt) - { - case AV_SAMPLE_FMT_FLTP: - { + switch (audio_codec->sample_fmt) { + case AV_SAMPLE_FMT_FLTP: { output_sample_fmt = AV_SAMPLE_FMT_FLT; break; } - case AV_SAMPLE_FMT_S32P: - { + case AV_SAMPLE_FMT_S32P: { output_sample_fmt = AV_SAMPLE_FMT_S32; break; } - case AV_SAMPLE_FMT_S16P: - { + case AV_SAMPLE_FMT_S16P: { output_sample_fmt = AV_SAMPLE_FMT_S16; break; } - case AV_SAMPLE_FMT_U8P: - { + case AV_SAMPLE_FMT_U8P: { output_sample_fmt = AV_SAMPLE_FMT_U8; break; } @@ -1325,45 +1487,46 @@ void FFmpegWriter::write_audio_packets(bool final) // setup resample context if (!avr) { avr = SWR_ALLOC(); - av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0); + av_opt_set_int(avr, "in_channel_layout", channel_layout_in_frame, 0); av_opt_set_int(avr, "out_channel_layout", info.channel_layout, 0); - av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0); - av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here - av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0); - av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0); - av_opt_set_int(avr, "in_channels", channels_in_frame, 0); - av_opt_set_int(avr, "out_channels", info.channels, 0); + av_opt_set_int(avr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0); + av_opt_set_int(avr, "out_sample_fmt", output_sample_fmt, 0); // planar not allowed here + av_opt_set_int(avr, "in_sample_rate", sample_rate_in_frame, 0); + av_opt_set_int(avr, "out_sample_rate", info.sample_rate, 0); + av_opt_set_int(avr, "in_channels", channels_in_frame, 0); + av_opt_set_int(avr, "out_channels", info.channels, 0); SWR_INIT(avr); } int nb_samples = 0; // Convert audio samples - nb_samples = SWR_CONVERT(avr, // audio resample context - audio_converted->data, // output data pointers - audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown) - audio_converted->nb_samples, // maximum number of samples that the output buffer can hold - audio_frame->data, // input data pointers - audio_frame->linesize[0], // input plane size, in bytes (0 if unknown) - audio_frame->nb_samples); // number of input samples to convert + nb_samples = SWR_CONVERT(avr, // audio resample context + audio_converted->data, // output data pointers + audio_converted->linesize[0], // output plane size, in bytes. (0 if unknown) + audio_converted->nb_samples, // maximum number of samples that the output buffer can hold + audio_frame->data, // input data pointers + audio_frame->linesize[0], // input plane size, in bytes (0 if unknown) + audio_frame->nb_samples); // number of input samples to convert // Create a new array (to hold all resampled S16 audio samples) - all_resampled_samples = (int16_t*)av_malloc(sizeof(int16_t) * nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))); + all_resampled_samples = (int16_t *) av_malloc( + sizeof(int16_t) * nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))); // Copy audio samples over original samples memcpy(all_resampled_samples, audio_converted->data[0], nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt)); // Remove converted audio av_freep(&(audio_frame->data[0])); - AV_FREE_FRAME(&audio_frame); + AV_FREE_FRAME(&audio_frame); av_freep(&audio_converted->data[0]); - AV_FREE_FRAME(&audio_converted); + AV_FREE_FRAME(&audio_converted); all_queued_samples = NULL; // this array cleared with above call ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 1st resampling)", "nb_samples", nb_samples, "remaining_frame_samples", remaining_frame_samples, "", -1, "", -1, "", -1, "", -1); } // Loop until no more samples - while (remaining_frame_samples > 0 || final) { + while (remaining_frame_samples > 0 || is_final) { // Get remaining samples needed for this packet int remaining_packet_samples = (audio_input_frame_size * info.channels) - audio_input_position; @@ -1375,9 +1538,10 @@ void FFmpegWriter::write_audio_packets(bool final) diff = remaining_frame_samples; // Copy frame samples into the packet samples array - if (!final) + if (!is_final) //TODO: Make this more sane - memcpy(samples + (audio_input_position * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))), all_resampled_samples + samples_position, diff * av_get_bytes_per_sample(output_sample_fmt)); + memcpy(samples + (audio_input_position * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))), + all_resampled_samples + samples_position, diff * av_get_bytes_per_sample(output_sample_fmt)); // Increment counters audio_input_position += diff; @@ -1386,28 +1550,27 @@ void FFmpegWriter::write_audio_packets(bool final) remaining_packet_samples -= diff; // Do we have enough samples to proceed? - if (audio_input_position < (audio_input_frame_size * info.channels) && !final) + if (audio_input_position < (audio_input_frame_size * info.channels) && !is_final) // Not enough samples to encode... so wait until the next frame break; // Convert to planar (if needed by audio codec) AVFrame *frame_final = AV_ALLOCATE_FRAME(); AV_RESET_FRAME(frame_final); - if (av_sample_fmt_is_planar(audio_codec->sample_fmt)) - { + if (av_sample_fmt_is_planar(audio_codec->sample_fmt)) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (2nd resampling for Planar formats)", "in_sample_fmt", output_sample_fmt, "out_sample_fmt", audio_codec->sample_fmt, "in_sample_rate", info.sample_rate, "out_sample_rate", info.sample_rate, "in_channels", info.channels, "out_channels", info.channels); // setup resample context if (!avr_planar) { avr_planar = SWR_ALLOC(); - av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0); + av_opt_set_int(avr_planar, "in_channel_layout", info.channel_layout, 0); av_opt_set_int(avr_planar, "out_channel_layout", info.channel_layout, 0); - av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0); - av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec->sample_fmt, 0); // planar not allowed here - av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0); - av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0); - av_opt_set_int(avr_planar, "in_channels", info.channels, 0); - av_opt_set_int(avr_planar, "out_channels", info.channels, 0); + av_opt_set_int(avr_planar, "in_sample_fmt", output_sample_fmt, 0); + av_opt_set_int(avr_planar, "out_sample_fmt", audio_codec->sample_fmt, 0); // planar not allowed here + av_opt_set_int(avr_planar, "in_sample_rate", info.sample_rate, 0); + av_opt_set_int(avr_planar, "out_sample_rate", info.sample_rate, 0); + av_opt_set_int(avr_planar, "in_channels", info.channels, 0); + av_opt_set_int(avr_planar, "out_channels", info.channels, 0); SWR_INIT(avr_planar); } @@ -1417,42 +1580,44 @@ void FFmpegWriter::write_audio_packets(bool final) audio_frame->nb_samples = audio_input_position / info.channels; // Create a new array - final_samples_planar = (int16_t*)av_malloc(sizeof(int16_t) * audio_frame->nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))); + final_samples_planar = (int16_t *) av_malloc( + sizeof(int16_t) * audio_frame->nb_samples * info.channels * (av_get_bytes_per_sample(output_sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))); // Copy audio into buffer for frame memcpy(final_samples_planar, samples, audio_frame->nb_samples * info.channels * av_get_bytes_per_sample(output_sample_fmt)); // Fill input frame with sample data avcodec_fill_audio_frame(audio_frame, info.channels, output_sample_fmt, (uint8_t *) final_samples_planar, - audio_encoder_buffer_size, 0); + audio_encoder_buffer_size, 0); // Create output frame (and allocate arrays) frame_final->nb_samples = audio_input_frame_size; av_samples_alloc(frame_final->data, frame_final->linesize, info.channels, frame_final->nb_samples, audio_codec->sample_fmt, 0); // Convert audio samples - int nb_samples = SWR_CONVERT(avr_planar, // audio resample context - frame_final->data, // output data pointers - frame_final->linesize[0], // output plane size, in bytes. (0 if unknown) - frame_final->nb_samples, // maximum number of samples that the output buffer can hold - audio_frame->data, // input data pointers - audio_frame->linesize[0], // input plane size, in bytes (0 if unknown) - audio_frame->nb_samples); // number of input samples to convert + int nb_samples = SWR_CONVERT(avr_planar, // audio resample context + frame_final->data, // output data pointers + frame_final->linesize[0], // output plane size, in bytes. (0 if unknown) + frame_final->nb_samples, // maximum number of samples that the output buffer can hold + audio_frame->data, // input data pointers + audio_frame->linesize[0], // input plane size, in bytes (0 if unknown) + audio_frame->nb_samples); // number of input samples to convert // Copy audio samples over original samples if (nb_samples > 0) memcpy(samples, frame_final->data[0], nb_samples * av_get_bytes_per_sample(audio_codec->sample_fmt) * info.channels); // deallocate AVFrame - av_freep(&(audio_frame->data[0])); - AV_FREE_FRAME(&audio_frame); - all_queued_samples = NULL; // this array cleared with above call + av_freep(&(audio_frame->data[0])); + AV_FREE_FRAME(&audio_frame); + all_queued_samples = NULL; // this array cleared with above call ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets (Successfully completed 2nd resampling for Planar formats)", "nb_samples", nb_samples, "", -1, "", -1, "", -1, "", -1, "", -1); } else { // Create a new array - final_samples = (int16_t*)av_malloc(sizeof(int16_t) * audio_input_position * (av_get_bytes_per_sample(audio_codec->sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))); + final_samples = (int16_t *) av_malloc( + sizeof(int16_t) * audio_input_position * (av_get_bytes_per_sample(audio_codec->sample_fmt) / av_get_bytes_per_sample(AV_SAMPLE_FMT_S16))); // Copy audio into buffer for frame memcpy(final_samples, samples, audio_input_position * av_get_bytes_per_sample(audio_codec->sample_fmt)); @@ -1462,7 +1627,7 @@ void FFmpegWriter::write_audio_packets(bool final) // Fill the final_frame AVFrame with audio (non planar) avcodec_fill_audio_frame(frame_final, audio_codec->channels, audio_codec->sample_fmt, (uint8_t *) final_samples, - audio_encoder_buffer_size, 0); + audio_encoder_buffer_size, 0); } // Increment PTS (in samples) @@ -1481,7 +1646,7 @@ void FFmpegWriter::write_audio_packets(bool final) /* encode the audio samples */ int got_packet_ptr = 0; - #if IS_FFMPEG_3_2 +#if IS_FFMPEG_3_2 // Encode audio (latest version of FFmpeg) int error_code; int ret = 0; @@ -1509,10 +1674,10 @@ void FFmpegWriter::write_audio_packets(bool final) ret = -1; } got_packet_ptr = ret; - #else +#else // Encode audio (older versions of FFmpeg) int error_code = avcodec_encode_audio2(audio_codec, &pkt, frame_final, &got_packet_ptr); - #endif +#endif /* if zero size, it means the image was buffered */ if (error_code == 0 && got_packet_ptr) { @@ -1534,27 +1699,25 @@ void FFmpegWriter::write_audio_packets(bool final) /* write the compressed frame in the media file */ int error_code = av_interleaved_write_frame(oc, &pkt); - if (error_code < 0) - { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); + if (error_code < 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string) av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); } } - if (error_code < 0) - { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); + if (error_code < 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_audio_packets ERROR [" + (string) av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); } // deallocate AVFrame av_freep(&(frame_final->data[0])); - AV_FREE_FRAME(&frame_final); + AV_FREE_FRAME(&frame_final); // deallocate memory for packet AV_FREE_PACKET(&pkt); // Reset position audio_input_position = 0; - final = false; + is_final = false; } // Delete arrays (if needed) @@ -1571,8 +1734,7 @@ void FFmpegWriter::write_audio_packets(bool final) } // Allocate an AVFrame object -AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer) -{ +AVFrame *FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer) { // Create an RGB AVFrame AVFrame *new_av_frame = NULL; @@ -1585,10 +1747,9 @@ AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int heig *buffer_size = AV_GET_IMAGE_SIZE(pix_fmt, width, height); // Create buffer (if not provided) - if (!new_buffer) - { + if (!new_buffer) { // New Buffer - new_buffer = (uint8_t*)av_malloc(*buffer_size * sizeof(uint8_t)); + new_buffer = (uint8_t *) av_malloc(*buffer_size * sizeof(uint8_t)); // Attach buffer to AVFrame AV_COPY_PICTURE_DATA(new_av_frame, new_buffer, pix_fmt, width, height); new_av_frame->width = width; @@ -1601,8 +1762,7 @@ AVFrame* FFmpegWriter::allocate_avframe(PixelFormat pix_fmt, int width, int heig } // process video frame -void FFmpegWriter::process_video_packet(std::shared_ptr frame) -{ +void FFmpegWriter::process_video_packet(std::shared_ptr frame) { // Determine the height & width of the source image int source_image_width = frame->GetWidth(); int source_image_height = frame->GetHeight(); @@ -1621,7 +1781,7 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) if (rescaler_position == num_of_rescalers) rescaler_position = 0; - #pragma omp task firstprivate(frame, scaler, source_image_width, source_image_height) +#pragma omp task firstprivate(frame, scaler, source_image_width, source_image_height) { // Allocate an RGB frame & final output frame int bytes_source = 0; @@ -1633,23 +1793,28 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) pixels = frame->GetPixels(); // Init AVFrame for source image & final (converted image) - frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t*) pixels); - #if IS_FFMPEG_3_2 - AVFrame *frame_final = allocate_avframe((AVPixelFormat)(video_st->codecpar->format), info.width, info.height, &bytes_final, NULL); - #else + frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t *) pixels); +#if IS_FFMPEG_3_2 + AVFrame *frame_final; + if (hw_en_on && hw_en_supported) { + frame_final = allocate_avframe(AV_PIX_FMT_NV12, info.width, info.height, &bytes_final, NULL); + } else { + frame_final = allocate_avframe((AVPixelFormat)(video_st->codecpar->format), info.width, info.height, &bytes_final, NULL); + } +#else AVFrame *frame_final = allocate_avframe(video_codec->pix_fmt, info.width, info.height, &bytes_final, NULL); - #endif +#endif // Fill with data - AV_COPY_PICTURE_DATA(frame_source, (uint8_t*)pixels, PIX_FMT_RGBA, source_image_width, source_image_height); + AV_COPY_PICTURE_DATA(frame_source, (uint8_t *) pixels, PIX_FMT_RGBA, source_image_width, source_image_height); ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::process_video_packet", "frame->number", frame->number, "bytes_source", bytes_source, "bytes_final", bytes_final, "", -1, "", -1, "", -1); // Resize & convert pixel format sws_scale(scaler, frame_source->data, frame_source->linesize, 0, - source_image_height, frame_final->data, frame_final->linesize); + source_image_height, frame_final->data, frame_final->linesize); // Add resized AVFrame to av_frames map - #pragma omp critical (av_frames_section) +#pragma omp critical (av_frames_section) add_avframe(frame, frame_final); // Deallocate memory @@ -1660,8 +1825,7 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) } // write video frame -bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* frame_final) -{ +bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *frame_final) { #if (LIBAVFORMAT_VERSION_MAJOR >= 58) ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet", "frame->number", frame->number, "oc->oformat->flags", oc->oformat->flags, "", -1, "", -1, "", -1, "", -1); #else @@ -1673,19 +1837,18 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* fra av_init_packet(&pkt); pkt.flags |= AV_PKT_FLAG_KEY; - pkt.stream_index= video_st->index; - pkt.data= (uint8_t*)frame_final->data; - pkt.size= sizeof(AVPicture); + pkt.stream_index = video_st->index; + pkt.data = (uint8_t *) frame_final->data; + pkt.size = sizeof(AVPicture); // Increment PTS (in frames and scaled to the codec's timebase) - write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base); + write_video_count += av_rescale_q(1, (AVRational) {info.fps.den, info.fps.num}, video_codec->time_base); pkt.pts = write_video_count; /* write the compressed frame in the media file */ int error_code = av_interleaved_write_frame(oc, &pkt); - if (error_code < 0) - { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); + if (error_code < 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string) av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); return false; } @@ -1694,7 +1857,7 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* fra } else #endif - { + { AVPacket pkt; av_init_packet(&pkt); @@ -1706,34 +1869,60 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* fra uint8_t *video_outbuf = NULL; // Increment PTS (in frames and scaled to the codec's timebase) - write_video_count += av_rescale_q(1, (AVRational){info.fps.den, info.fps.num}, video_codec->time_base); + write_video_count += av_rescale_q(1, (AVRational) {info.fps.den, info.fps.num}, video_codec->time_base); // Assign the initial AVFrame PTS from the frame counter frame_final->pts = write_video_count; - +#if IS_FFMPEG_3_2 + if (hw_en_on && hw_en_supported) { + if (!(hw_frame = av_frame_alloc())) { + fprintf(stderr, "Error code: av_hwframe_alloc\n"); + } + if (av_hwframe_get_buffer(video_codec->hw_frames_ctx, hw_frame, 0) < 0) { + fprintf(stderr, "Error code: av_hwframe_get_buffer\n"); + } + if (!hw_frame->hw_frames_ctx) { + fprintf(stderr, "Error hw_frames_ctx.\n"); + } + hw_frame->format = AV_PIX_FMT_NV12; + if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) { + fprintf(stderr, "Error while transferring frame data to surface.\n"); + } + av_frame_copy_props(hw_frame, frame_final); + } +#endif /* encode the image */ int got_packet_ptr = 0; int error_code = 0; - #if IS_FFMPEG_3_2 +#if IS_FFMPEG_3_2 // Write video packet (latest version of FFmpeg) int frameFinished = 0; - int ret = avcodec_send_frame(video_codec, frame_final); + int ret; + + if (hw_en_on && hw_en_supported) { + ret = avcodec_send_frame(video_codec, hw_frame); //hw_frame!!! + } else { + ret = avcodec_send_frame(video_codec, frame_final); + } error_code = ret; if (ret < 0 ) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet (Frame not sent)", "", -1, "", -1, "", -1, "", -1, "", -1, "", -1); - if (ret == AVERROR(EAGAIN) ) + if (ret == AVERROR(EAGAIN) ) { cerr << "Frame EAGAIN" << "\n"; - if (ret == AVERROR_EOF ) + } + if (ret == AVERROR_EOF ) { cerr << "Frame AVERROR_EOF" << "\n"; + } avcodec_send_frame(video_codec, NULL); } else { while (ret >= 0) { ret = avcodec_receive_packet(video_codec, &pkt); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { avcodec_flush_buffers(video_codec); got_packet_ptr = 0; - break; + break; } if (ret == 0) { got_packet_ptr = 1; @@ -1741,34 +1930,36 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* fra } } } - #else - #if LIBAVFORMAT_VERSION_MAJOR >= 54 - // Write video packet (older than FFmpeg 3.2) - error_code = avcodec_encode_video2(video_codec, &pkt, frame_final, &got_packet_ptr); - if (error_code != 0 ) - cerr << "Frame AVERROR_EOF" << "\n"; - if (got_packet_ptr == 0 ) - cerr << "Frame gotpacket error" << "\n"; - #else - // Write video packet (even older versions of FFmpeg) - int video_outbuf_size = 200000; - video_outbuf = (uint8_t*) av_malloc(200000); +#else +#if LIBAVFORMAT_VERSION_MAJOR >= 54 + // Write video packet (older than FFmpeg 3.2) + error_code = avcodec_encode_video2(video_codec, &pkt, frame_final, &got_packet_ptr); + if (error_code != 0) { + cerr << "Frame AVERROR_EOF" << "\n"; + } + if (got_packet_ptr == 0) { + cerr << "Frame gotpacket error" << "\n"; + } +#else + // Write video packet (even older versions of FFmpeg) + int video_outbuf_size = 200000; + video_outbuf = (uint8_t*) av_malloc(200000); - /* encode the image */ - int out_size = avcodec_encode_video(video_codec, video_outbuf, video_outbuf_size, frame_final); + /* encode the image */ + int out_size = avcodec_encode_video(video_codec, video_outbuf, video_outbuf_size, frame_final); - /* if zero size, it means the image was buffered */ - if (out_size > 0) { - if(video_codec->coded_frame->key_frame) - pkt.flags |= AV_PKT_FLAG_KEY; - pkt.data= video_outbuf; - pkt.size= out_size; + /* if zero size, it means the image was buffered */ + if (out_size > 0) { + if(video_codec->coded_frame->key_frame) + pkt.flags |= AV_PKT_FLAG_KEY; + pkt.data= video_outbuf; + pkt.size= out_size; - // got data back (so encode this frame) - got_packet_ptr = 1; - } - #endif - #endif + // got data back (so encode this frame) + got_packet_ptr = 1; + } +#endif +#endif /* if zero size, it means the image was buffered */ if (error_code == 0 && got_packet_ptr) { @@ -1788,9 +1979,8 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* fra /* write the compressed frame in the media file */ int error_code = av_interleaved_write_frame(oc, &pkt); - if (error_code < 0) - { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string)av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); + if (error_code < 0) { + ZmqLogger::Instance()->AppendDebugMethod("FFmpegWriter::write_video_packet ERROR [" + (string) av_err2str(error_code) + "]", "error_code", error_code, "", -1, "", -1, "", -1, "", -1, "", -1); return false; } } @@ -1801,6 +1991,14 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* fra // Deallocate packet AV_FREE_PACKET(&pkt); +#if IS_FFMPEG_3_2 + if (hw_en_on && hw_en_supported) { + if (hw_frame) { + av_frame_free(&hw_frame); + hw_frame = NULL; + } + } +#endif } // Success @@ -1808,25 +2006,30 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame* fra } // Output the ffmpeg info about this format, streams, and codecs (i.e. dump format) -void FFmpegWriter::OutputStreamInfo() -{ +void FFmpegWriter::OutputStreamInfo() { // output debug info av_dump_format(oc, 0, path.c_str(), 1); } // Init a collection of software rescalers (thread safe) -void FFmpegWriter::InitScalers(int source_width, int source_height) -{ +void FFmpegWriter::InitScalers(int source_width, int source_height) { int scale_mode = SWS_FAST_BILINEAR; if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) { scale_mode = SWS_LANCZOS; } // Init software rescalers vector (many of them, one for each thread) - for (int x = 0; x < num_of_rescalers; x++) - { + for (int x = 0; x < num_of_rescalers; x++) { // Init the software scaler from FFMpeg - img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(video_st, video_st->codec), scale_mode, NULL, NULL, NULL); +#if IS_FFMPEG_3_2 + if (hw_en_on && hw_en_supported) { + img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, AV_PIX_FMT_NV12, SWS_BILINEAR, NULL, NULL, NULL); + } else +#endif + { + img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(video_st, video_st->codec), SWS_BILINEAR, + NULL, NULL, NULL); + } // Add rescaler to vector image_rescalers.push_back(img_convert_ctx); @@ -1840,8 +2043,7 @@ void FFmpegWriter::ResampleAudio(int sample_rate, int channels) { } // Remove & deallocate all software scalers -void FFmpegWriter::RemoveScalers() -{ +void FFmpegWriter::RemoveScalers() { // Close all rescalers for (int x = 0; x < num_of_rescalers; x++) sws_freeContext(image_rescalers[x]); diff --git a/src/KeyFrame.cpp b/src/KeyFrame.cpp index a2c2362d..735763e5 100644 --- a/src/KeyFrame.cpp +++ b/src/KeyFrame.cpp @@ -327,18 +327,13 @@ bool Keyframe::IsIncreasing(int index) } } - if (current_value < next_value) { - // Increasing - return true; - } - else if (current_value >= next_value) { + if (current_value >= next_value) { // Decreasing return false; } } - else - // return default true (since most curves increase) - return true; + // return default true (since most curves increase) + return true; } // Generate JSON string of this object diff --git a/src/Qt/AudioPlaybackThread.cpp b/src/Qt/AudioPlaybackThread.cpp index c64bd688..7bad4649 100644 --- a/src/Qt/AudioPlaybackThread.cpp +++ b/src/Qt/AudioPlaybackThread.cpp @@ -35,25 +35,44 @@ namespace openshot AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::m_pInstance = NULL; // Create or Get an instance of the device manager singleton - AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance(int numChannels) + AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance() { if (!m_pInstance) { // Create the actual instance of device manager only once m_pInstance = new AudioDeviceManagerSingleton; + // Get preferred audio device name (if any) + juce::String preferred_audio_device = juce::String(Settings::Instance()->PLAYBACK_AUDIO_DEVICE_NAME.c_str()); + // Initialize audio device only 1 time - String error = m_pInstance->audioDeviceManager.initialise ( + juce::String audio_error = m_pInstance->audioDeviceManager.initialise ( 0, /* number of input channels */ - numChannels, /* number of output channels */ + 2, /* number of output channels */ 0, /* no XML settings.. */ - true /* select default device on failure */); + true, /* select default device on failure */ + preferred_audio_device /* preferredDefaultDeviceName */); // Persist any errors detected - if (error.isNotEmpty()) { - m_pInstance->initialise_error = error.toStdString(); + if (audio_error.isNotEmpty()) { + m_pInstance->initialise_error = audio_error.toRawUTF8(); } else { m_pInstance->initialise_error = ""; } + + // Get all audio device names + for (int i = 0; i < m_pInstance->audioDeviceManager.getAvailableDeviceTypes().size(); ++i) + { + const AudioIODeviceType* t = m_pInstance->audioDeviceManager.getAvailableDeviceTypes()[i]; + const StringArray deviceNames = t->getDeviceNames (); + + for (int j = 0; j < deviceNames.size (); ++j ) + { + juce::String deviceName = deviceNames[j]; + juce::String typeName = t->getTypeName(); + AudioDeviceInfo deviceInfo = {deviceName.toRawUTF8(), typeName.toRawUTF8()}; + m_pInstance->audio_device_names.push_back(deviceInfo); + } + } } return m_pInstance; @@ -149,7 +168,7 @@ namespace openshot // Start new audio device (or get existing one) // Add callback - AudioDeviceManagerSingleton::Instance(numChannels)->audioDeviceManager.addAudioCallback(&player); + AudioDeviceManagerSingleton::Instance()->audioDeviceManager.addAudioCallback(&player); // Create TimeSliceThread for audio buffering time_thread.startThread(); @@ -182,7 +201,7 @@ namespace openshot transport.setSource(NULL); player.setSource(NULL); - AudioDeviceManagerSingleton::Instance(0)->audioDeviceManager.removeAudioCallback(&player); + AudioDeviceManagerSingleton::Instance()->audioDeviceManager.removeAudioCallback(&player); // Remove source delete source; diff --git a/src/QtImageReader.cpp b/src/QtImageReader.cpp index 8cc2debc..fbd53006 100644 --- a/src/QtImageReader.cpp +++ b/src/QtImageReader.cpp @@ -130,6 +130,10 @@ void QtImageReader::Open() info.display_ratio.num = size.num; info.display_ratio.den = size.den; + // Set current max size + max_size.setWidth(info.width); + max_size.setHeight(info.height); + // Mark as "open" is_open = true; } @@ -209,8 +213,7 @@ std::shared_ptr QtImageReader::GetFrame(int64_t requested_frame) } // Scale image smaller (or use a previous scaled image) - if (!cached_image || (cached_image->width() != max_width || cached_image->height() != max_height)) { - + if (!cached_image || (max_size.width() != max_width || max_size.height() != max_height)) { #if USE_RESVG == 1 // If defined and found in CMake, utilize the libresvg for parsing // SVG files and rasterizing them to QImages. @@ -239,6 +242,10 @@ std::shared_ptr QtImageReader::GetFrame(int64_t requested_frame) cached_image = std::shared_ptr(new QImage(image->scaled(max_width, max_height, Qt::KeepAspectRatio, Qt::SmoothTransformation))); cached_image = std::shared_ptr(new QImage(cached_image->convertToFormat(QImage::Format_RGBA8888))); #endif + + // Set max size (to later determine if max_size is changed) + max_size.setWidth(max_width); + max_size.setHeight(max_height); } // Create or get frame object diff --git a/src/QtPlayer.cpp b/src/QtPlayer.cpp index 4f53c7ca..3287c19d 100644 --- a/src/QtPlayer.cpp +++ b/src/QtPlayer.cpp @@ -56,7 +56,7 @@ QtPlayer::~QtPlayer() void QtPlayer::CloseAudioDevice() { // Close audio device (only do this once, when all audio playback is finished) - AudioDeviceManagerSingleton::Instance(0)->CloseAudioDevice(); + AudioDeviceManagerSingleton::Instance()->CloseAudioDevice(); } // Return any error string during initialization @@ -69,6 +69,15 @@ string QtPlayer::GetError() { } } +/// Get Audio Devices from JUCE +vector QtPlayer::GetAudioDeviceNames() { + if (reader && threads_started) { + return p->audioPlayback->getAudioDeviceNames(); + } else { + return vector(); + } +} + void QtPlayer::SetSource(const std::string &source) { FFmpegReader *ffreader = new FFmpegReader(source); diff --git a/src/Settings.cpp b/src/Settings.cpp index b13f0f5a..8193ec6b 100644 --- a/src/Settings.cpp +++ b/src/Settings.cpp @@ -40,12 +40,18 @@ Settings *Settings::Instance() if (!m_pInstance) { // Create the actual instance of logger only once m_pInstance = new Settings; - m_pInstance->HARDWARE_DECODE = false; - m_pInstance->HARDWARE_ENCODE = false; + m_pInstance->HARDWARE_DECODER = 0; m_pInstance->HIGH_QUALITY_SCALING = false; m_pInstance->MAX_WIDTH = 0; m_pInstance->MAX_HEIGHT = 0; m_pInstance->WAIT_FOR_VIDEO_PROCESSING_TASK = false; + m_pInstance->OMP_THREADS = 12; + m_pInstance->FF_THREADS = 8; + m_pInstance->DE_LIMIT_HEIGHT_MAX = 1100; + m_pInstance->DE_LIMIT_WIDTH_MAX = 1950; + m_pInstance->HW_DE_DEVICE_SET = 0; + m_pInstance->HW_EN_DEVICE_SET = 0; + m_pInstance->PLAYBACK_AUDIO_DEVICE_NAME = ""; } return m_pInstance; diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 384273bd..b229a3de 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -58,6 +58,9 @@ Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int cha info.has_audio = true; info.has_video = true; info.video_length = info.fps.ToFloat() * info.duration; + info.display_ratio = openshot::Fraction(width, height); + info.display_ratio.Reduce(); + info.pixel_ratio = openshot::Fraction(1, 1); // Init max image size SetMaxSize(info.width, info.height); @@ -277,9 +280,10 @@ void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, in /* Apply effects to the source frame (if any). If multiple clips are overlapping, only process the * effects on the top clip. */ - if (is_top_clip && source_frame) + if (is_top_clip && source_frame) { #pragma omp critical (T_addLayer) source_frame = apply_effects(source_frame, timeline_frame_number, source_clip->Layer()); + } // Declare an image to hold the source frame's image std::shared_ptr source_image; @@ -1370,6 +1374,33 @@ void Timeline::apply_json_to_timeline(Json::Value change) { else if (root_key == "fps" && sub_key == "den") // Set fps.den info.fps.den = change["value"].asInt(); + else if (root_key == "display_ratio" && sub_key == "" && change["value"].isObject()) { + // Set display_ratio fraction + if (!change["value"]["num"].isNull()) + info.display_ratio.num = change["value"]["num"].asInt(); + if (!change["value"]["den"].isNull()) + info.display_ratio.den = change["value"]["den"].asInt(); + } + else if (root_key == "display_ratio" && sub_key == "num") + // Set display_ratio.num + info.display_ratio.num = change["value"].asInt(); + else if (root_key == "display_ratio" && sub_key == "den") + // Set display_ratio.den + info.display_ratio.den = change["value"].asInt(); + else if (root_key == "pixel_ratio" && sub_key == "" && change["value"].isObject()) { + // Set pixel_ratio fraction + if (!change["value"]["num"].isNull()) + info.pixel_ratio.num = change["value"]["num"].asInt(); + if (!change["value"]["den"].isNull()) + info.pixel_ratio.den = change["value"]["den"].asInt(); + } + else if (root_key == "pixel_ratio" && sub_key == "num") + // Set pixel_ratio.num + info.pixel_ratio.num = change["value"].asInt(); + else if (root_key == "pixel_ratio" && sub_key == "den") + // Set pixel_ratio.den + info.pixel_ratio.den = change["value"].asInt(); + else if (root_key == "sample_rate") // Set sample rate info.sample_rate = change["value"].asInt(); @@ -1379,9 +1410,7 @@ void Timeline::apply_json_to_timeline(Json::Value change) { else if (root_key == "channel_layout") // Set channel layout info.channel_layout = (ChannelLayout) change["value"].asInt(); - else - // Error parsing JSON (or missing keys) throw InvalidJSONKey("JSON change key is invalid", change.toStyledString()); @@ -1442,7 +1471,14 @@ void Timeline::ClearAllCache() { // Set Max Image Size (used for performance optimization). Convenience function for setting // Settings::Instance()->MAX_WIDTH and Settings::Instance()->MAX_HEIGHT. void Timeline::SetMaxSize(int width, int height) { - // Init max image size (choose the smallest one) - Settings::Instance()->MAX_WIDTH = min(width, info.width); - Settings::Instance()->MAX_HEIGHT = min(height, info.height); + // Maintain aspect ratio regardless of what size is passed in + QSize display_ratio_size = QSize(info.display_ratio.num * info.pixel_ratio.ToFloat(), info.display_ratio.den * info.pixel_ratio.ToFloat()); + QSize proposed_size = QSize(min(width, info.width), min(height, info.height)); + + // Scale QSize up to proposed size + display_ratio_size.scale(proposed_size, Qt::KeepAspectRatio); + + // Set max size + Settings::Instance()->MAX_WIDTH = display_ratio_size.width(); + Settings::Instance()->MAX_HEIGHT = display_ratio_size.height(); } \ No newline at end of file diff --git a/src/bindings/python/CMakeLists.txt b/src/bindings/python/CMakeLists.txt index 3418d2de..2a481aa7 100644 --- a/src/bindings/python/CMakeLists.txt +++ b/src/bindings/python/CMakeLists.txt @@ -65,18 +65,29 @@ if (PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND) target_link_libraries(${SWIG_MODULE_pyopenshot_REAL_NAME} ${PYTHON_LIBRARIES} openshot) - ### FIND THE PYTHON INTERPRETER (AND THE SITE PACKAGES FOLDER) - execute_process ( COMMAND ${PYTHON_EXECUTABLE} -c "\ + ### Check if the following Debian-friendly python module path exists + SET(PYTHON_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/site-packages") + if (NOT EXISTS ${PYTHON_MODULE_PATH}) + + ### Check if another Debian-friendly python module path exists + SET(PYTHON_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/dist-packages") + if (NOT EXISTS ${PYTHON_MODULE_PATH}) + + ### Calculate the python module path (using distutils) + execute_process ( COMMAND ${PYTHON_EXECUTABLE} -c "\ from distutils.sysconfig import get_python_lib; \ print( get_python_lib( plat_specific=True, prefix='${CMAKE_INSTALL_PREFIX}' ) )" - OUTPUT_VARIABLE _ABS_PYTHON_MODULE_PATH - OUTPUT_STRIP_TRAILING_WHITESPACE ) + OUTPUT_VARIABLE _ABS_PYTHON_MODULE_PATH + OUTPUT_STRIP_TRAILING_WHITESPACE ) - GET_FILENAME_COMPONENT(_ABS_PYTHON_MODULE_PATH - "${_ABS_PYTHON_MODULE_PATH}" ABSOLUTE) - FILE(RELATIVE_PATH _REL_PYTHON_MODULE_PATH - ${CMAKE_INSTALL_PREFIX} ${_ABS_PYTHON_MODULE_PATH}) - SET(PYTHON_MODULE_PATH ${_REL_PYTHON_MODULE_PATH}) + GET_FILENAME_COMPONENT(_ABS_PYTHON_MODULE_PATH + "${_ABS_PYTHON_MODULE_PATH}" ABSOLUTE) + FILE(RELATIVE_PATH _REL_PYTHON_MODULE_PATH + ${CMAKE_INSTALL_PREFIX} ${_ABS_PYTHON_MODULE_PATH}) + SET(PYTHON_MODULE_PATH ${_ABS_PYTHON_MODULE_PATH}) + endif() + endif() + message("PYTHON_MODULE_PATH: ${PYTHON_MODULE_PATH}") ############### INSTALL HEADERS & LIBRARY ################ ### Install Python bindings diff --git a/src/bindings/python/openshot.i b/src/bindings/python/openshot.i index de1f020c..ed34b658 100644 --- a/src/bindings/python/openshot.i +++ b/src/bindings/python/openshot.i @@ -87,6 +87,7 @@ #include "../../../include/Settings.h" #include "../../../include/Timeline.h" #include "../../../include/ZmqLogger.h" +#include "../../../include/AudioDeviceInfo.h" %} @@ -154,6 +155,7 @@ %include "../../../include/Settings.h" %include "../../../include/Timeline.h" %include "../../../include/ZmqLogger.h" +%include "../../../include/AudioDeviceInfo.h" #ifdef USE_IMAGEMAGICK %include "../../../include/ImageReader.h" @@ -187,4 +189,5 @@ namespace std { %template(FieldVector) vector; %template(MappedFrameVector) vector; %template(MappedMetadata) map; + %template(AudioDeviceInfoVector) vector; } diff --git a/src/bindings/ruby/openshot.i b/src/bindings/ruby/openshot.i index b9a35d41..2868708e 100644 --- a/src/bindings/ruby/openshot.i +++ b/src/bindings/ruby/openshot.i @@ -57,6 +57,14 @@ namespace std { %{ +/* Ruby and FFmpeg define competing RSHIFT macros, + * so we move Ruby's out of the way for now. We'll + * restore it after dealing with FFmpeg's + */ +#ifdef RSHIFT + #define RB_RSHIFT(a, b) RSHIFT(a, b) + #undef RSHIFT +#endif #include "../../../include/Version.h" #include "../../../include/ReaderBase.h" #include "../../../include/WriterBase.h" @@ -91,7 +99,17 @@ namespace std { #include "../../../include/Settings.h" #include "../../../include/Timeline.h" #include "../../../include/ZmqLogger.h" +#include "../../../include/AudioDeviceInfo.h" +/* Move FFmpeg's RSHIFT to FF_RSHIFT, if present */ +#ifdef RSHIFT + #define FF_RSHIFT(a, b) RSHIFT(a, b) + #undef RSHIFT +#endif +/* And restore Ruby's RSHIFT, if we captured it */ +#ifdef RB_RSHIFT + #define RSHIFT(a, b) RB_RSHIFT(a, b) +#endif %} #ifdef USE_BLACKMAGIC @@ -132,8 +150,29 @@ namespace std { %include "../../../include/EffectInfo.h" %include "../../../include/Enums.h" %include "../../../include/Exceptions.h" + +/* Ruby and FFmpeg define competing RSHIFT macros, + * so we move Ruby's out of the way for now. We'll + * restore it after dealing with FFmpeg's + */ +#ifdef RSHIFT + #define RB_RSHIFT(a, b) RSHIFT(a, b) + #undef RSHIFT +#endif + %include "../../../include/FFmpegReader.h" %include "../../../include/FFmpegWriter.h" + +/* Move FFmpeg's RSHIFT to FF_RSHIFT, if present */ +#ifdef RSHIFT + #define FF_RSHIFT(a, b) RSHIFT(a, b) + #undef RSHIFT +#endif +/* And restore Ruby's RSHIFT, if we captured it */ +#ifdef RB_RSHIFT + #define RSHIFT(a, b) RB_RSHIFT(a, b) +#endif + %include "../../../include/Fraction.h" %include "../../../include/Frame.h" %include "../../../include/FrameMapper.h" @@ -147,6 +186,7 @@ namespace std { %include "../../../include/Settings.h" %include "../../../include/Timeline.h" %include "../../../include/ZmqLogger.h" +%include "../../../include/AudioDeviceInfo.h" #ifdef USE_IMAGEMAGICK %include "../../../include/ImageReader.h" @@ -181,4 +221,5 @@ namespace std { %template(FieldVector) vector; %template(MappedFrameVector) vector; %template(MappedMetadata) map; + %template(AudioDeviceInfoVector) vector; } diff --git a/src/examples/Example.cpp b/src/examples/Example.cpp index 411abdda..80339684 100644 --- a/src/examples/Example.cpp +++ b/src/examples/Example.cpp @@ -36,6 +36,10 @@ using namespace openshot; int main(int argc, char* argv[]) { + Settings *s = Settings::Instance(); + s->HARDWARE_DECODER = 2; // 1 VA-API, 2 NVDEC + s->HW_DE_DEVICE_SET = 1; + FFmpegReader r9("/home/jonathan/Videos/sintel_trailer-720p.mp4"); r9.Open(); r9.DisplayInfo(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 72884f1f..e358fb44 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,8 +6,8 @@ # # Copyright (c) 2008-2014 OpenShot Studios, LLC # . This file is part of -# OpenShot Library (libopenshot), an open-source project dedicated to -# delivering high quality video editing and animation solutions to the +# OpenShot Library (libopenshot), an open-source project dedicated to +# delivering high quality video editing and animation solutions to the # world. For more information visit . # # OpenShot Library (libopenshot) is free software: you can redistribute it @@ -135,7 +135,7 @@ add_definitions(${Qt5Gui_DEFINITIONS}) add_definitions(${Qt5Multimedia_DEFINITIONS}) add_definitions(${Qt5MultimediaWidgets_DEFINITIONS}) -SET(QT_LIBRARIES ${Qt5Widgets_LIBRARIES} +SET(QT_LIBRARIES ${Qt5Widgets_LIBRARIES} ${Qt5Core_LIBRARIES} ${Qt5Gui_LIBRARIES} ${Qt5Multimedia_LIBRARIES} @@ -157,7 +157,7 @@ qt5_wrap_cpp(MOC_FILES ${QT_HEADER_FILES}) IF (ENABLE_BLACKMAGIC) # Find BlackMagic DeckLinkAPI libraries FIND_PACKAGE(BlackMagic) - + IF (BLACKMAGIC_FOUND) # Include Blackmagic headers (needed for compile) include_directories(${BLACKMAGIC_INCLUDE_DIR}) diff --git a/tests/Settings_Tests.cpp b/tests/Settings_Tests.cpp index 86790653..1b8180c9 100644 --- a/tests/Settings_Tests.cpp +++ b/tests/Settings_Tests.cpp @@ -36,8 +36,7 @@ TEST(Settings_Default_Constructor) // Create an empty color Settings *s = Settings::Instance(); - CHECK_EQUAL(false, s->HARDWARE_DECODE); - CHECK_EQUAL(false, s->HARDWARE_ENCODE); + CHECK_EQUAL(0, s->HARDWARE_DECODER); CHECK_EQUAL(false, s->HIGH_QUALITY_SCALING); CHECK_EQUAL(false, s->WAIT_FOR_VIDEO_PROCESSING_TASK); } @@ -46,18 +45,15 @@ TEST(Settings_Change_Settings) { // Create an empty color Settings *s = Settings::Instance(); - s->HARDWARE_DECODE = true; - s->HARDWARE_ENCODE = true; + s->HARDWARE_DECODER = 1; s->HIGH_QUALITY_SCALING = true; s->WAIT_FOR_VIDEO_PROCESSING_TASK = true; - CHECK_EQUAL(true, s->HARDWARE_DECODE); - CHECK_EQUAL(true, s->HARDWARE_ENCODE); + CHECK_EQUAL(1, s->HARDWARE_DECODER); CHECK_EQUAL(true, s->HIGH_QUALITY_SCALING); CHECK_EQUAL(true, s->WAIT_FOR_VIDEO_PROCESSING_TASK); - CHECK_EQUAL(true, s->HARDWARE_DECODE); - CHECK_EQUAL(true, s->HARDWARE_ENCODE); + CHECK_EQUAL(1, s->HARDWARE_DECODER); CHECK_EQUAL(true, Settings::Instance()->HIGH_QUALITY_SCALING); CHECK_EQUAL(true, Settings::Instance()->WAIT_FOR_VIDEO_PROCESSING_TASK); -} \ No newline at end of file +}