From 76fb7c8013daa5113638518378a679ad24c20629 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Tue, 24 Aug 2021 04:21:55 -0400 Subject: [PATCH 1/8] Rename AudioSampleBuffer to AudioBuffer For a long time now, juce::AudioSampleBuffer has been nothing but an alias for its true (templated) type, AudioBuffer. --- bindings/python/openshot.i | 24 ++++++++++++++--------- bindings/ruby/openshot.i | 32 +++++++++++++++++------------- src/AudioBufferSource.cpp | 4 ++-- src/AudioBufferSource.h | 6 +++--- src/AudioReaderSource.cpp | 10 +++++----- src/AudioReaderSource.h | 6 +++--- src/AudioResampler.cpp | 8 ++++---- src/AudioResampler.h | 10 +++++----- src/Clip.cpp | 36 +++++++++++++++++----------------- src/Clip.h | 2 +- src/Frame.cpp | 14 ++++++------- src/Frame.h | 6 +++--- src/audio_effects/Compressor.h | 10 +++++----- src/audio_effects/Delay.h | 10 +++++----- src/audio_effects/Echo.h | 12 ++++++------ src/audio_effects/Expander.h | 10 +++++----- src/audio_effects/STFT.cpp | 10 +++++----- src/audio_effects/STFT.h | 14 ++++++------- 18 files changed, 117 insertions(+), 107 deletions(-) diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index 6f3f61c4..6f386ee7 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -54,13 +54,23 @@ #ifdef USE_IMAGEMAGICK %shared_ptr(Magick::Image) #endif -%shared_ptr(juce::AudioSampleBuffer) +%shared_ptr(juce::AudioBuffer) %shared_ptr(openshot::Frame) +/* Instantiate the required template specializations */ +%template() std::map; +%template() std::pair; +%template() std::vector; +%template() std::pair; +%template() std::pair; +%template() std::pair; +%template() std::vector>; + %{ #include "OpenShotVersion.h" #include "ReaderBase.h" #include "WriterBase.h" +#include "AudioDevices.h" #include "CacheBase.h" #include "CacheDisk.h" #include "CacheMemory.h" @@ -95,7 +105,6 @@ #include "TimelineBase.h" #include "Timeline.h" #include "ZmqLogger.h" -#include "AudioDeviceInfo.h" %} @@ -136,12 +145,7 @@ } } -/* Instantiate the required template specializations */ -%template() std::map; -%template() std::pair; -%template() std::vector; -%template() std::pair; -%template() std::pair; + /* Wrap std templates (list, vector, etc...) */ %template(ClipList) std::list; @@ -151,6 +155,8 @@ %template(FieldVector) std::vector; %template(MappedFrameVector) std::vector; %template(MetadataMap) std::map; + +/* Deprecated */ %template(AudioDeviceInfoVector) std::vector; /* Make openshot.Fraction more Pythonic */ @@ -251,6 +257,7 @@ %include "OpenShotVersion.h" %include "ReaderBase.h" %include "WriterBase.h" +%include "AudioDevices.h" %include "CacheBase.h" %include "CacheDisk.h" %include "CacheMemory.h" @@ -289,7 +296,6 @@ %include "TimelineBase.h" %include "Timeline.h" %include "ZmqLogger.h" -%include "AudioDeviceInfo.h" #ifdef USE_OPENCV %include "ClipProcessingJobs.h" diff --git a/bindings/ruby/openshot.i b/bindings/ruby/openshot.i index 3e64af26..882b5e7c 100644 --- a/bindings/ruby/openshot.i +++ b/bindings/ruby/openshot.i @@ -54,15 +54,17 @@ #ifdef USE_IMAGEMAGICK %shared_ptr(Magick::Image) #endif -%shared_ptr(juce::AudioSampleBuffer) +%shared_ptr(juce::AudioBuffer) %shared_ptr(openshot::Frame) -/* Template specializations */ +/* Instantiate the required template specializations */ %template() std::map; %template() std::pair; %template() std::vector; %template() std::pair; %template() std::pair; +%template() std::pair; +%template() std::vector>; %{ /* Ruby and FFmpeg define competing RSHIFT macros, @@ -76,6 +78,7 @@ #include "OpenShotVersion.h" #include "ReaderBase.h" #include "WriterBase.h" +#include "AudioDevices.h" #include "CacheBase.h" #include "CacheDisk.h" #include "CacheMemory.h" @@ -110,7 +113,6 @@ #include "TimelineBase.h" #include "Timeline.h" #include "ZmqLogger.h" -#include "AudioDeviceInfo.h" /* Move FFmpeg's RSHIFT to FF_RSHIFT, if present */ #ifdef RSHIFT @@ -138,9 +140,22 @@ %} #endif +/* Wrap std templates (list, vector, etc...) */ +%template(ClipList) std::list; +%template(EffectBaseList) std::list; +%template(CoordinateVector) std::vector; +%template(PointsVector) std::vector; +%template(FieldVector) std::vector; +%template(MappedFrameVector) std::vector; +%template(MetadataMap) std::map; + +/* Deprecated */ +%template(AudioDeviceInfoVector) std::vector; + %include "OpenShotVersion.h" %include "ReaderBase.h" %include "WriterBase.h" +%include "AudioDevices.h" %include "CacheBase.h" %include "CacheDisk.h" %include "CacheMemory.h" @@ -200,7 +215,6 @@ %include "TimelineBase.h" %include "Timeline.h" %include "ZmqLogger.h" -%include "AudioDeviceInfo.h" #ifdef USE_IMAGEMAGICK %include "ImageReader.h" @@ -208,7 +222,6 @@ %include "TextReader.h" #endif - /* Effects */ %include "effects/Bars.h" %include "effects/Blur.h" @@ -227,12 +240,3 @@ %include "effects/Wave.h" -/* Wrap std templates (list, vector, etc...) */ -%template(ClipList) std::list; -%template(EffectBaseList) std::list; -%template(CoordinateVector) std::vector; -%template(PointsVector) std::vector; -%template(FieldVector) std::vector; -%template(MappedFrameVector) std::vector; -%template(MetadataMap) std::map; -%template(AudioDeviceInfoVector) std::vector; diff --git a/src/AudioBufferSource.cpp b/src/AudioBufferSource.cpp index 6b2bc59e..4a8c0f51 100644 --- a/src/AudioBufferSource.cpp +++ b/src/AudioBufferSource.cpp @@ -34,7 +34,7 @@ using namespace std; using namespace openshot; // Default constructor -AudioBufferSource::AudioBufferSource(juce::AudioSampleBuffer *audio_buffer) +AudioBufferSource::AudioBufferSource(juce::AudioBuffer *audio_buffer) : position(0), repeat(false), buffer(audio_buffer) { } @@ -134,7 +134,7 @@ void AudioBufferSource::setLooping (bool shouldLoop) } // Use a different AudioSampleBuffer for this source -void AudioBufferSource::setBuffer (juce::AudioSampleBuffer *audio_buffer) +void AudioBufferSource::setBuffer (juce::AudioBuffer *audio_buffer) { buffer = audio_buffer; setNextReadPosition(0); diff --git a/src/AudioBufferSource.h b/src/AudioBufferSource.h index a899d8da..12d1e09e 100644 --- a/src/AudioBufferSource.h +++ b/src/AudioBufferSource.h @@ -49,12 +49,12 @@ namespace openshot private: int position; bool repeat; - juce::AudioSampleBuffer *buffer; + juce::AudioBuffer *buffer; public: /// @brief Default constructor /// @param audio_buffer This buffer contains the samples you want to play through JUCE. - AudioBufferSource(juce::AudioSampleBuffer *audio_buffer); + AudioBufferSource(juce::AudioBuffer *audio_buffer); /// Destructor ~AudioBufferSource(); @@ -87,7 +87,7 @@ namespace openshot void setLooping (bool shouldLoop); /// Update the internal buffer used by this source - void setBuffer (juce::AudioSampleBuffer *audio_buffer); + void setBuffer (juce::AudioBuffer *audio_buffer); }; } diff --git a/src/AudioReaderSource.cpp b/src/AudioReaderSource.cpp index acd69deb..4ff6ae97 100644 --- a/src/AudioReaderSource.cpp +++ b/src/AudioReaderSource.cpp @@ -40,7 +40,7 @@ AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_ size(buffer_size), position(0), frame_position(0), estimated_frame(0), speed(1) { // Initialize an audio buffer (based on reader) - buffer = new juce::AudioSampleBuffer(reader->info.channels, size); + buffer = new juce::AudioBuffer(reader->info.channels, size); // initialize the audio samples to zero (silence) buffer->clear(); @@ -73,7 +73,7 @@ void AudioReaderSource::GetMoreSamplesFromReader() estimated_frame = frame_number; // Init new buffer - juce::AudioSampleBuffer *new_buffer = new juce::AudioSampleBuffer(reader->info.channels, size); + auto *new_buffer = new juce::AudioBuffer(reader->info.channels, size); new_buffer->clear(); // Move the remaining samples into new buffer (if any) @@ -142,7 +142,7 @@ void AudioReaderSource::GetMoreSamplesFromReader() } // Reverse an audio buffer -juce::AudioSampleBuffer* AudioReaderSource::reverse_buffer(juce::AudioSampleBuffer* buffer) +juce::AudioBuffer* AudioReaderSource::reverse_buffer(juce::AudioBuffer* buffer) { int number_of_samples = buffer->getNumSamples(); int channels = buffer->getNumChannels(); @@ -151,7 +151,7 @@ juce::AudioSampleBuffer* AudioReaderSource::reverse_buffer(juce::AudioSampleBuff ZmqLogger::Instance()->AppendDebugMethod("AudioReaderSource::reverse_buffer", "number_of_samples", number_of_samples, "channels", channels); // Reverse array (create new buffer to hold the reversed version) - juce::AudioSampleBuffer *reversed = new juce::AudioSampleBuffer(channels, number_of_samples); + auto *reversed = new juce::AudioBuffer(channels, number_of_samples); reversed->clear(); for (int channel = 0; channel < channels; channel++) @@ -286,7 +286,7 @@ void AudioReaderSource::setLooping (bool shouldLoop) } // Update the internal buffer used by this source -void AudioReaderSource::setBuffer (juce::AudioSampleBuffer *audio_buffer) +void AudioReaderSource::setBuffer (juce::AudioBuffer *audio_buffer) { buffer = audio_buffer; setNextReadPosition(0); diff --git a/src/AudioReaderSource.h b/src/AudioReaderSource.h index 2b0cc0ea..54665697 100644 --- a/src/AudioReaderSource.h +++ b/src/AudioReaderSource.h @@ -50,7 +50,7 @@ namespace openshot int position; /// The position of the audio source (index of buffer) bool repeat; /// Repeat the audio source when finished int size; /// The size of the internal buffer - juce::AudioSampleBuffer *buffer; /// The audio sample buffer + juce::AudioBuffer *buffer; /// The audio sample buffer int speed; /// The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) ReaderBase *reader; /// The reader to pull samples from @@ -64,7 +64,7 @@ namespace openshot void GetMoreSamplesFromReader(); /// Reverse an audio buffer (for backwards audio) - juce::AudioSampleBuffer* reverse_buffer(juce::AudioSampleBuffer* buffer); + juce::AudioBuffer* reverse_buffer(juce::AudioBuffer* buffer); public: @@ -105,7 +105,7 @@ namespace openshot void setLooping (bool shouldLoop); /// Update the internal buffer used by this source - void setBuffer (juce::AudioSampleBuffer *audio_buffer); + void setBuffer (juce::AudioBuffer *audio_buffer); const ReaderInfo & getReaderInfo() const { return reader->info; } diff --git a/src/AudioResampler.cpp b/src/AudioResampler.cpp index 8c27043e..2c7183fa 100644 --- a/src/AudioResampler.cpp +++ b/src/AudioResampler.cpp @@ -52,7 +52,7 @@ AudioResampler::AudioResampler() resample_source = new juce::ResamplingAudioSource(buffer_source, false, 2); // Init resampled buffer - resampled_buffer = new juce::AudioSampleBuffer(2, 1); + resampled_buffer = new juce::AudioBuffer(2, 1); resampled_buffer->clear(); // Init callback buffer @@ -74,7 +74,7 @@ AudioResampler::~AudioResampler() } // Sets the audio buffer and updates the key settings -void AudioResampler::SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate) +void AudioResampler::SetBuffer(juce::AudioBuffer *new_buffer, double sample_rate, double new_sample_rate) { if (sample_rate <= 0) sample_rate = 44100; @@ -89,7 +89,7 @@ void AudioResampler::SetBuffer(juce::AudioSampleBuffer *new_buffer, double sampl } // Sets the audio buffer and key settings -void AudioResampler::SetBuffer(juce::AudioSampleBuffer *new_buffer, double ratio) +void AudioResampler::SetBuffer(juce::AudioBuffer *new_buffer, double ratio) { // Update buffer & buffer source buffer = new_buffer; @@ -120,7 +120,7 @@ void AudioResampler::SetBuffer(juce::AudioSampleBuffer *new_buffer, double ratio } // Get the resampled audio buffer -juce::AudioSampleBuffer* AudioResampler::GetResampledBuffer() +juce::AudioBuffer* AudioResampler::GetResampledBuffer() { // Resample the current frame's audio buffer (into the temp callback buffer) resample_source->getNextAudioBlock(resample_callback_buffer); diff --git a/src/AudioResampler.h b/src/AudioResampler.h index f9f94d83..dc758a04 100644 --- a/src/AudioResampler.h +++ b/src/AudioResampler.h @@ -44,8 +44,8 @@ namespace openshot { */ class AudioResampler { private: - juce::AudioSampleBuffer *buffer; - juce::AudioSampleBuffer *resampled_buffer; + juce::AudioBuffer *buffer; + juce::AudioBuffer *resampled_buffer; openshot::AudioBufferSource *buffer_source; juce::ResamplingAudioSource *resample_source; juce::AudioSourceChannelInfo resample_callback_buffer; @@ -67,15 +67,15 @@ namespace openshot { /// @param new_buffer The buffer of audio samples needing to be resampled /// @param sample_rate The original sample rate of the buffered samples /// @param new_sample_rate The requested sample rate you need - void SetBuffer(juce::AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate); + void SetBuffer(juce::AudioBuffer *new_buffer, double sample_rate, double new_sample_rate); /// @brief Sets the audio buffer and key settings /// @param new_buffer The buffer of audio samples needing to be resampled /// @param ratio The multiplier that needs to be applied to the sample rate (this is how resampling happens) - void SetBuffer(juce::AudioSampleBuffer *new_buffer, double ratio); + void SetBuffer(juce::AudioBuffer *new_buffer, double ratio); /// Get the resampled audio buffer - juce::AudioSampleBuffer* GetResampledBuffer(); + juce::AudioBuffer* GetResampledBuffer(); }; } diff --git a/src/Clip.cpp b/src/Clip.cpp index 35d0fba1..e4d486f3 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -466,13 +466,13 @@ std::string Clip::get_file_extension(std::string path) } // Reverse an audio buffer -void Clip::reverse_buffer(juce::AudioSampleBuffer* buffer) +void Clip::reverse_buffer(juce::AudioBuffer* buffer) { int number_of_samples = buffer->getNumSamples(); int channels = buffer->getNumChannels(); // Reverse array (create new buffer to hold the reversed version) - juce::AudioSampleBuffer *reversed = new juce::AudioSampleBuffer(channels, number_of_samples); + auto *reversed = new juce::AudioBuffer(channels, number_of_samples); reversed->clear(); for (int channel = 0; channel < channels; channel++) @@ -490,7 +490,7 @@ void Clip::reverse_buffer(juce::AudioSampleBuffer* buffer) buffer->addFrom(channel, 0, reversed->getReadPointer(channel), number_of_samples, 1.0f); delete reversed; - reversed = NULL; + reversed = nullptr; } // Adjust the audio and image of a time mapped frame @@ -507,7 +507,7 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num const GenericScopedLock lock(getFrameCriticalSection); // create buffer and resampler - juce::AudioSampleBuffer *samples = NULL; + juce::AudioBuffer *samples = nullptr; if (!resampler) resampler = new AudioResampler(); @@ -527,10 +527,10 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num if (time.GetRepeatFraction(frame_number).den > 1) { // SLOWING DOWN AUDIO // Resample data, and return new buffer pointer - juce::AudioSampleBuffer *resampled_buffer = NULL; + juce::AudioBuffer *resampled_buffer = nullptr; // SLOW DOWN audio (split audio) - samples = new juce::AudioSampleBuffer(channels, number_of_samples); + samples = new juce::AudioBuffer(channels, number_of_samples); samples->clear(); // Loop through channels, and get audio samples @@ -559,7 +559,7 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num number_of_samples, 1.0f); // Clean up - resampled_buffer = NULL; + resampled_buffer = nullptr; } else if (abs(delta) > 1 && abs(delta) < 100) { @@ -574,7 +574,7 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num reader->info.channels); // Allocate a new sample buffer for these delta frames - samples = new juce::AudioSampleBuffer(channels, total_delta_samples); + samples = new juce::AudioBuffer(channels, total_delta_samples); samples->clear(); // Loop through each frame in this delta @@ -582,8 +582,8 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num delta_frame <= new_frame_number; delta_frame++) { // buffer to hold detal samples int number_of_delta_samples = GetOrCreateFrame(delta_frame)->GetAudioSamplesCount(); - juce::AudioSampleBuffer *delta_samples = new juce::AudioSampleBuffer(channels, - number_of_delta_samples); + auto *delta_samples = new juce::AudioBuffer(channels, + number_of_delta_samples); delta_samples->clear(); for (int channel = 0; channel < channels; channel++) @@ -602,7 +602,7 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num // Clean up delete delta_samples; - delta_samples = NULL; + delta_samples = nullptr; // Increment start position start += number_of_delta_samples; @@ -618,7 +618,7 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num reader->info.channels); // Allocate a new sample buffer for these delta frames - samples = new juce::AudioSampleBuffer(channels, total_delta_samples); + samples = new juce::AudioBuffer(channels, total_delta_samples); samples->clear(); // Loop through each frame in this delta @@ -626,8 +626,8 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num delta_frame >= new_frame_number; delta_frame--) { // buffer to hold delta samples int number_of_delta_samples = GetOrCreateFrame(delta_frame)->GetAudioSamplesCount(); - juce::AudioSampleBuffer *delta_samples = new juce::AudioSampleBuffer(channels, - number_of_delta_samples); + auto *delta_samples = new juce::AudioBuffer(channels, + number_of_delta_samples); delta_samples->clear(); for (int channel = 0; channel < channels; channel++) @@ -657,7 +657,7 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num resampler->SetBuffer(samples, float(start) / float(number_of_samples)); // Resample data, and return new buffer pointer - juce::AudioSampleBuffer *buffer = resampler->GetResampledBuffer(); + juce::AudioBuffer *buffer = resampler->GetResampledBuffer(); // Add the newly resized audio samples to the current frame for (int channel = 0; channel < channels; channel++) @@ -665,11 +665,11 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num frame->AddAudio(true, channel, 0, buffer->getReadPointer(channel), number_of_samples, 1.0f); // Clean up - buffer = NULL; + buffer = nullptr; } else { // Use the samples on this frame (but maybe reverse them if needed) - samples = new juce::AudioSampleBuffer(channels, number_of_samples); + samples = new juce::AudioBuffer(channels, number_of_samples); samples->clear(); // Loop through channels, and get audio samples @@ -689,7 +689,7 @@ void Clip::get_time_mapped_frame(std::shared_ptr frame, int64_t frame_num } delete samples; - samples = NULL; + samples = nullptr; } } } diff --git a/src/Clip.h b/src/Clip.h index 2b9caced..5e2769a1 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -167,7 +167,7 @@ namespace openshot { void sort_effects(); /// Reverse an audio buffer - void reverse_buffer(juce::AudioSampleBuffer* buffer); + void reverse_buffer(juce::AudioBuffer* buffer); public: diff --git a/src/Frame.cpp b/src/Frame.cpp index 244b065f..f97f93ed 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -53,7 +53,7 @@ using namespace openshot; // Constructor - image & audio Frame::Frame(int64_t number, int width, int height, std::string color, int samples, int channels) - : audio(std::make_shared(channels, samples)), + : audio(std::make_shared>(channels, samples)), number(number), width(width), height(height), pixel_ratio(1,1), color(color), qbuffer(NULL), channels(channels), channel_layout(LAYOUT_STEREO), @@ -111,7 +111,7 @@ void Frame::DeepCopy(const Frame& other) if (other.image) image = std::make_shared(*(other.image)); if (other.audio) - audio = std::make_shared(*(other.audio)); + audio = std::make_shared>(*(other.audio)); if (other.wave_image) wave_image = std::make_shared(*(other.wave_image)); } @@ -324,8 +324,8 @@ float* Frame::GetAudioSamples(int channel) // Get a planar array of sample data, using any sample rate float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count) { - float *output = NULL; - juce::AudioSampleBuffer *buffer(audio.get()); + float *output = nullptr; + juce::AudioBuffer *buffer(audio.get()); int num_of_channels = audio->getNumChannels(); int num_of_samples = GetAudioSamplesCount(); @@ -370,8 +370,8 @@ float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampl // Get an array of sample data (all channels interleaved together), using any sample rate float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count) { - float *output = NULL; - juce::AudioSampleBuffer *buffer(audio.get()); + float *output = nullptr; + juce::AudioBuffer *buffer(audio.get()); int num_of_channels = audio->getNumChannels(); int num_of_samples = GetAudioSamplesCount(); @@ -429,7 +429,7 @@ int Frame::GetAudioSamplesCount() return max_audio_sample; } -juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer() +juce::AudioBuffer *Frame::GetAudioSampleBuffer() { return audio.get(); } diff --git a/src/Frame.h b/src/Frame.h index e35ea9ca..5e16bad3 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -109,7 +109,7 @@ namespace openshot private: std::shared_ptr image; std::shared_ptr wave_image; - + std::shared_ptr previewApp; juce::CriticalSection addingImageSection; juce::CriticalSection addingAudioSection; @@ -131,7 +131,7 @@ namespace openshot int constrain(int color_value); public: - std::shared_ptr audio; + std::shared_ptr> audio; int64_t number; ///< This is the frame number (starting at 1) bool has_audio_data; ///< This frame has been loaded with audio data bool has_image_data; ///< This frame has been loaded with pixel data @@ -224,7 +224,7 @@ namespace openshot /// Get number of audio samples int GetAudioSamplesCount(); - juce::AudioSampleBuffer *GetAudioSampleBuffer(); + juce::AudioBuffer *GetAudioSampleBuffer(); /// Get the size in bytes of this frame (rough estimate) int64_t GetBytes(); diff --git a/src/audio_effects/Compressor.h b/src/audio_effects/Compressor.h index 7dcdb252..32ce1440 100644 --- a/src/audio_effects/Compressor.h +++ b/src/audio_effects/Compressor.h @@ -1,7 +1,7 @@ /** * @file * @brief Header file for Compressor audio effect class - * @author + * @author * * @ref License */ @@ -55,7 +55,7 @@ namespace openshot private: /// Init effect settings void init_effect_details(); - + public: Keyframe threshold; @@ -65,7 +65,7 @@ namespace openshot Keyframe makeup_gain; Keyframe bypass; - juce::AudioSampleBuffer mixed_down_input; + juce::AudioBuffer mixed_down_input; float xl; float yl; float xg; @@ -94,8 +94,8 @@ namespace openshot /// /// @returns A new openshot::Frame object /// @param frame_number The frame number (starting at 1) of the clip or effect on the timeline. - std::shared_ptr GetFrame(int64_t frame_number) override { - return GetFrame(std::make_shared(), frame_number); + std::shared_ptr GetFrame(int64_t frame_number) override { + return GetFrame(std::make_shared(), frame_number); } /// @brief This method is required for all derived classes of ClipBase, and returns a diff --git a/src/audio_effects/Delay.h b/src/audio_effects/Delay.h index a693c292..68f9c8fb 100644 --- a/src/audio_effects/Delay.h +++ b/src/audio_effects/Delay.h @@ -1,7 +1,7 @@ /** * @file * @brief Header file for Delay audio effect class - * @author + * @author * * @ref License */ @@ -57,9 +57,9 @@ namespace openshot void init_effect_details(); public: - Keyframe delay_time; + Keyframe delay_time; - juce::AudioSampleBuffer delay_buffer; + juce::AudioBuffer delay_buffer; int delay_buffer_samples; int delay_buffer_channels; int delay_write_position; @@ -77,8 +77,8 @@ namespace openshot /// /// @returns A new openshot::Frame object /// @param frame_number The frame number (starting at 1) of the clip or effect on the timeline. - std::shared_ptr GetFrame(int64_t frame_number) override { - return GetFrame(std::make_shared(), frame_number); + std::shared_ptr GetFrame(int64_t frame_number) override { + return GetFrame(std::make_shared(), frame_number); } void setup(std::shared_ptr frame); diff --git a/src/audio_effects/Echo.h b/src/audio_effects/Echo.h index 9a120b6e..68212afe 100644 --- a/src/audio_effects/Echo.h +++ b/src/audio_effects/Echo.h @@ -1,7 +1,7 @@ /** * @file * @brief Header file for Echo audio effect class - * @author + * @author * * @ref License */ @@ -57,11 +57,11 @@ namespace openshot void init_effect_details(); public: - Keyframe echo_time; - Keyframe feedback; + Keyframe echo_time; + Keyframe feedback; Keyframe mix; - juce::AudioSampleBuffer echo_buffer; + juce::AudioBuffer echo_buffer; int echo_buffer_samples; int echo_buffer_channels; int echo_write_position; @@ -79,8 +79,8 @@ namespace openshot /// /// @returns A new openshot::Frame object /// @param frame_number The frame number (starting at 1) of the clip or effect on the timeline. - std::shared_ptr GetFrame(int64_t frame_number) override { - return GetFrame(std::make_shared(), frame_number); + std::shared_ptr GetFrame(int64_t frame_number) override { + return GetFrame(std::make_shared(), frame_number); } void setup(std::shared_ptr frame); diff --git a/src/audio_effects/Expander.h b/src/audio_effects/Expander.h index 4eee84af..3542911a 100644 --- a/src/audio_effects/Expander.h +++ b/src/audio_effects/Expander.h @@ -1,7 +1,7 @@ /** * @file * @brief Header file for Expander audio effect class - * @author + * @author * * @ref License */ @@ -55,7 +55,7 @@ namespace openshot private: /// Init effect settings void init_effect_details(); - + public: Keyframe threshold; @@ -65,7 +65,7 @@ namespace openshot Keyframe makeup_gain; Keyframe bypass; - juce::AudioSampleBuffer mixed_down_input; + juce::AudioBuffer mixed_down_input; float xl; float yl; float xg; @@ -94,8 +94,8 @@ namespace openshot /// /// @returns A new openshot::Frame object /// @param frame_number The frame number (starting at 1) of the clip or effect on the timeline. - std::shared_ptr GetFrame(int64_t frame_number) override { - return GetFrame(std::make_shared(), frame_number); + std::shared_ptr GetFrame(int64_t frame_number) override { + return GetFrame(std::make_shared(), frame_number); } /// @brief This method is required for all derived classes of ClipBase, and returns a diff --git a/src/audio_effects/STFT.cpp b/src/audio_effects/STFT.cpp index 3b87e6e3..fa83feab 100644 --- a/src/audio_effects/STFT.cpp +++ b/src/audio_effects/STFT.cpp @@ -14,7 +14,7 @@ void STFT::updateParameters(const int new_fft_size, const int new_overlap, const updateWindow(new_window_type); } -void STFT::process(juce::AudioSampleBuffer &block) +void STFT::process(juce::AudioBuffer &block) { num_samples = block.getNumSamples(); @@ -25,7 +25,7 @@ void STFT::process(juce::AudioSampleBuffer &block) current_output_buffer_write_position = output_buffer_write_position; current_output_buffer_read_position = output_buffer_read_position; current_samples_since_last_FFT = samples_since_last_FFT; - + for (int sample = 0; sample < num_samples; ++sample) { const float input_sample = channel_data[sample]; @@ -78,7 +78,7 @@ void STFT::updateFftSize(const int new_fft_size) frequency_domain_buffer.realloc(fft_size); frequency_domain_buffer.clear(fft_size); - + input_buffer_write_position = 0; output_buffer_write_position = 0; output_buffer_read_position = 0; @@ -126,7 +126,7 @@ void STFT::updateWindow(const int new_window_type) break; } } - + float window_sum = 0.0f; for (int sample = 0; sample < fft_size; ++sample) window_sum += fft_window[sample]; @@ -185,4 +185,4 @@ void STFT::synthesis(const int channel) current_output_buffer_write_position += hop_size; if (current_output_buffer_write_position >= output_buffer_length) current_output_buffer_write_position = 0; -} \ No newline at end of file +} diff --git a/src/audio_effects/STFT.h b/src/audio_effects/STFT.h index 71f71a78..1e346a91 100644 --- a/src/audio_effects/STFT.h +++ b/src/audio_effects/STFT.h @@ -14,23 +14,23 @@ namespace openshot { public: STFT() : num_channels (1) { } - + virtual ~STFT() { } void setup(const int num_input_channels); - void process(juce::AudioSampleBuffer &block); + void process(juce::AudioBuffer &block); void updateParameters(const int new_fft_size, const int new_overlap, const int new_window_type); virtual void updateFftSize(const int new_fft_size); - + virtual void updateHopSize(const int new_overlap); virtual void updateWindow(const int new_window_type); - + private: - + virtual void modification(const int channel); virtual void analysis(const int channel); @@ -45,10 +45,10 @@ namespace openshot std::unique_ptr fft; int input_buffer_length; - juce::AudioSampleBuffer input_buffer; + juce::AudioBuffer input_buffer; int output_buffer_length; - juce::AudioSampleBuffer output_buffer; + juce::AudioBuffer output_buffer; juce::HeapBlock fft_window; juce::HeapBlock> time_domain_buffer; From 6aab6fd957dee83423125b6d80336a0ed0425230 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Tue, 24 Aug 2021 13:03:46 -0400 Subject: [PATCH 2/8] Audio: New device name lookup - A new AudioDevices class replaces the AudioDeviceInfo struct. It has a single method, getNames(), which: * creates an AudioDeviceManager (NOT using the singleton) * scans for available devices * returns the results as a std::vector containing std::pair objects (The AudioDevices device manager is never initialize()d, so no devices are opened; it should be safe to use even DURING playback, without disruption.) By using STL containers (rather than a custom struct) to return the results, Python is able to consume the output as a native list of tuples. AudioDeviceInfo is still present for compatibility, but deprecated. - Eliminated some unnecessary conversions (like): * calls to std::string::c_str, when passing to juce::String. juce::String accepts std::string directly. * calls to juce::String::toRawUTF8, when creating std::string. There's a juce::String::ToStdString, which is better. --- src/AudioDevices.cpp | 59 +++++++++++++++++++++++ src/{AudioDeviceInfo.h => AudioDevices.h} | 30 +++++++++--- src/CMakeLists.txt | 1 + src/Qt/AudioPlaybackThread.cpp | 59 ++++++++++++++--------- src/Qt/AudioPlaybackThread.h | 55 +++++++++++---------- src/QtPlayer.cpp | 18 ++++--- src/QtPlayer.h | 5 +- src/Settings.cpp | 1 + src/Settings.h | 3 ++ 9 files changed, 167 insertions(+), 64 deletions(-) create mode 100644 src/AudioDevices.cpp rename src/{AudioDeviceInfo.h => AudioDevices.h} (74%) diff --git a/src/AudioDevices.cpp b/src/AudioDevices.cpp new file mode 100644 index 00000000..fc6282e8 --- /dev/null +++ b/src/AudioDevices.cpp @@ -0,0 +1,59 @@ +/** + * @file + * @brief Utility methods for identifying audio devices + * @author Jonathan Thomas + * @author FeRD (Frank Dana) + * + * @ref License + */ + +/* LICENSE + * + * Copyright (c) 2008-2021 OpenShot Studios, LLC + * . This file is part of + * OpenShot Library (libopenshot), an open-source project dedicated to + * delivering high quality video editing and animation solutions to the + * world. For more information visit . + * + * OpenShot Library (libopenshot) is free software: you can redistribute it + * and/or modify it under the terms of the GNU Lesser General Public License + * as published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * OpenShot Library (libopenshot) is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with OpenShot Library. If not, see . + */ + +#include "AudioDevices.h" + +#include + +using namespace openshot; + +using AudioDeviceList = std::vector>; + +// Build a list of devices found, and return +AudioDeviceList AudioDevices::getNames() { + // A temporary device manager, used to scan device names. + // Its initialize() is never called, and devices are not opened. + std::unique_ptr + manager(new juce::AudioDeviceManager()); + + m_devices.clear(); + + auto &types = manager->getAvailableDeviceTypes(); + for (auto* t : types) { + t->scanForDevices(); + const auto names = t->getDeviceNames(); + for (const auto& name : names) { + m_devices.emplace_back( + name.toStdString(), t->getTypeName().toStdString()); + } + } + return m_devices; +} diff --git a/src/AudioDeviceInfo.h b/src/AudioDevices.h similarity index 74% rename from src/AudioDeviceInfo.h rename to src/AudioDevices.h index 454bf923..f1565506 100644 --- a/src/AudioDeviceInfo.h +++ b/src/AudioDevices.h @@ -31,17 +31,35 @@ #ifndef OPENSHOT_AUDIODEVICEINFO_H #define OPENSHOT_AUDIODEVICEINFO_H +#include +#include +namespace openshot { /** * @brief This struct hold information about Audio Devices * * The type and name of the audio device. */ -namespace openshot { - struct AudioDeviceInfo - { - std::string name; - std::string type; - }; +struct +AudioDeviceInfo { + std::string name; + std::string type; +}; + +using AudioDeviceList = std::vector>; + + /// A class which probes the available audio devices +class AudioDevices +{ +public: + AudioDevices() = default; + + /// Return a vector of std::pair<> objects holding the + /// device name and type for each audio device detected + AudioDeviceList getNames(); +private: + AudioDeviceList m_devices; +}; + } #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15a7bcc8..1cc803b9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,6 +61,7 @@ add_feature_info("IWYU (include-what-you-use)" ENABLE_IWYU "Scan all source file # Main library sources set(OPENSHOT_SOURCES AudioBufferSource.cpp + AudioDevices.cpp AudioReaderSource.cpp AudioResampler.cpp CacheBase.cpp diff --git a/src/Qt/AudioPlaybackThread.cpp b/src/Qt/AudioPlaybackThread.cpp index 178642f6..e93a75fe 100644 --- a/src/Qt/AudioPlaybackThread.cpp +++ b/src/Qt/AudioPlaybackThread.cpp @@ -31,6 +31,12 @@ #include "AudioPlaybackThread.h" +#include "../ReaderBase.h" +#include "../RendererBase.h" +#include "../AudioReaderSource.h" +#include "../AudioDevices.h" +#include "../Settings.h" + #include // for std::this_thread::sleep_for #include // for std::chrono::milliseconds @@ -46,39 +52,46 @@ namespace openshot if (!m_pInstance) { // Create the actual instance of device manager only once m_pInstance = new AudioDeviceManagerSingleton; + auto* mgr = &m_pInstance->audioDeviceManager; - // Get preferred audio device name (if any) - juce::String preferred_audio_device = juce::String(Settings::Instance()->PLAYBACK_AUDIO_DEVICE_NAME.c_str()); + // Get preferred audio device name and type (if any) + auto selected_device = juce::String( + Settings::Instance()->PLAYBACK_AUDIO_DEVICE_NAME); + auto selected_type = juce::String( + Settings::Instance()->PLAYBACK_AUDIO_DEVICE_TYPE); + + if (selected_type.isEmpty() && !selected_device.isEmpty()) { + // Look up type for the selected device + for (const auto t : mgr->getAvailableDeviceTypes()) { + for (const auto n : t->getDeviceNames()) { + if (selected_device.trim().equalsIgnoreCase(n.trim())) { + selected_type = t->getTypeName(); + break; + } + } + if(!selected_type.isEmpty()) + break; + } + } + + if (!selected_type.isEmpty()) + m_pInstance->audioDeviceManager.setCurrentAudioDeviceType(selected_type, true); // Initialize audio device only 1 time juce::String audio_error = m_pInstance->audioDeviceManager.initialise ( - 0, /* number of input channels */ - 2, /* number of output channels */ - 0, /* no XML settings.. */ - true, /* select default device on failure */ - preferred_audio_device /* preferredDefaultDeviceName */); + 0, // number of input channels + 2, // number of output channels + nullptr, // no XML settings.. + true, // select default device on failure + selected_device // preferredDefaultDeviceName + ); // Persist any errors detected if (audio_error.isNotEmpty()) { - m_pInstance->initialise_error = audio_error.toRawUTF8(); + m_pInstance->initialise_error = audio_error.toStdString(); } else { m_pInstance->initialise_error = ""; } - - // Get all audio device names - for (int i = 0; i < m_pInstance->audioDeviceManager.getAvailableDeviceTypes().size(); ++i) - { - const AudioIODeviceType* t = m_pInstance->audioDeviceManager.getAvailableDeviceTypes()[i]; - const juce::StringArray deviceNames = t->getDeviceNames (); - - for (int j = 0; j < deviceNames.size (); ++j ) - { - juce::String deviceName = deviceNames[j]; - juce::String typeName = t->getTypeName(); - openshot::AudioDeviceInfo deviceInfo = {deviceName.toRawUTF8(), typeName.toRawUTF8()}; - m_pInstance->audio_device_names.push_back(deviceInfo); - } - } } return m_pInstance; diff --git a/src/Qt/AudioPlaybackThread.h b/src/Qt/AudioPlaybackThread.h index bf9aaf32..b19c9c16 100644 --- a/src/Qt/AudioPlaybackThread.h +++ b/src/Qt/AudioPlaybackThread.h @@ -32,33 +32,35 @@ #ifndef OPENSHOT_AUDIO_PLAYBACK_THREAD_H #define OPENSHOT_AUDIO_PLAYBACK_THREAD_H -#include "../ReaderBase.h" -#include "../RendererBase.h" -#include "../AudioReaderSource.h" -#include "../AudioDeviceInfo.h" -#include "../Settings.h" +#include "AudioDevices.h" +#include "AudioReaderSource.h" + +#include namespace openshot { - /** - * @brief Singleton wrapper for AudioDeviceManager (to prevent multiple instances). - */ - class AudioDeviceManagerSingleton { - private: +// Forward decls +class ReaderBase; +class Frame; +class PlayerPrivate; +class QtPlayer; + +/** + * @brief Singleton wrapper for AudioDeviceManager (to prevent multiple instances). + */ +class AudioDeviceManagerSingleton { +private: /// Default constructor (Don't allow user to create an instance of this singleton) AudioDeviceManagerSingleton(){ initialise_error=""; }; /// Private variable to keep track of singleton instance static AudioDeviceManagerSingleton * m_pInstance; - public: +public: /// Error found during JUCE initialise method std::string initialise_error; - /// List of valid audio device names - std::vector audio_device_names; - /// Override with no channels and no preferred audio device static AudioDeviceManagerSingleton * Instance(); @@ -117,22 +119,23 @@ namespace openshot /// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) int getSpeed() const { if (source) return source->getSpeed(); else return 1; } - /// Get Audio Error (if any) - std::string getError() - { - return AudioDeviceManagerSingleton::Instance()->initialise_error; - } + /// Get Audio Error (if any) + std::string getError() + { + return AudioDeviceManagerSingleton::Instance()->initialise_error; + } - /// Get Audio Device Names (if any) - std::vector getAudioDeviceNames() - { - return AudioDeviceManagerSingleton::Instance()->audio_device_names; - }; + /// Get Audio Device Names (if any) + std::vector> getAudioDeviceNames() + { + AudioDevices devs; + return devs.getNames(); + }; friend class PlayerPrivate; friend class QtPlayer; - }; +}; -} +} // namespace openshot #endif // OPENSHOT_AUDIO_PLAYBACK_THREAD_H diff --git a/src/QtPlayer.cpp b/src/QtPlayer.cpp index f3697f71..504f5a7f 100644 --- a/src/QtPlayer.cpp +++ b/src/QtPlayer.cpp @@ -29,15 +29,20 @@ * along with OpenShot Library. If not, see . */ +#include "QtPlayer.h" + +#include "AudioDevices.h" #include "Clip.h" #include "FFmpegReader.h" #include "Timeline.h" -#include "QtPlayer.h" #include "Qt/PlayerPrivate.h" #include "Qt/VideoRenderer.h" namespace openshot { + + using AudioDeviceList = std::vector>; + // Delegating constructor QtPlayer::QtPlayer() : QtPlayer::QtPlayer(new VideoRenderer()) @@ -77,12 +82,9 @@ namespace openshot } /// Get Audio Devices from JUCE - std::vector QtPlayer::GetAudioDeviceNames() { - if (reader && threads_started) { - return p->audioPlayback->getAudioDeviceNames(); - } else { - return std::vector(); - } + AudioDeviceList QtPlayer::GetAudioDeviceNames() { + AudioDevices devs; + return devs.getNames(); } void QtPlayer::SetSource(const std::string &source) @@ -237,4 +239,4 @@ namespace openshot void QtPlayer::Volume(float new_volume) { volume = new_volume; } -} \ No newline at end of file +} diff --git a/src/QtPlayer.h b/src/QtPlayer.h index 7ddf6ada..3bb1fbcf 100644 --- a/src/QtPlayer.h +++ b/src/QtPlayer.h @@ -34,12 +34,15 @@ #include #include + #include "PlayerBase.h" #include "Qt/PlayerPrivate.h" #include "RendererBase.h" namespace openshot { + using AudioDeviceList = std::vector>; + /** * @brief This class is used to playback a video from a reader. * @@ -64,7 +67,7 @@ namespace openshot std::string GetError(); /// Get Audio Devices from JUCE - std::vector GetAudioDeviceNames(); + AudioDeviceList GetAudioDeviceNames(); /// Play the video void Play(); diff --git a/src/Settings.cpp b/src/Settings.cpp index 688eaae3..4d2597e2 100644 --- a/src/Settings.cpp +++ b/src/Settings.cpp @@ -52,6 +52,7 @@ Settings *Settings::Instance() m_pInstance->HW_DE_DEVICE_SET = 0; m_pInstance->HW_EN_DEVICE_SET = 0; m_pInstance->PLAYBACK_AUDIO_DEVICE_NAME = ""; + m_pInstance->PLAYBACK_AUDIO_DEVICE_TYPE = ""; m_pInstance->DEBUG_TO_STDERR = false; auto env_debug = std::getenv("LIBOPENSHOT_DEBUG"); if (env_debug != nullptr) diff --git a/src/Settings.h b/src/Settings.h index e21822a0..5077dee5 100644 --- a/src/Settings.h +++ b/src/Settings.h @@ -103,6 +103,9 @@ namespace openshot { /// The audio device name to use during playback std::string PLAYBACK_AUDIO_DEVICE_NAME = ""; + /// The device type for the playback audio devices + std::string PLAYBACK_AUDIO_DEVICE_TYPE = ""; + /// The current install path of OpenShot (needs to be set when using Timeline(path), since certain /// paths depend on the location of OpenShot transitions and files) std::string PATH_OPENSHOT_INSTALL = ""; From ac3762923d7387a6c226f40519de56d3cf08a5ff Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 2 Dec 2021 22:10:08 -0600 Subject: [PATCH 3/8] Updating licensing for reuse compliance --- src/AudioDevices.cpp | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/AudioDevices.cpp b/src/AudioDevices.cpp index fc6282e8..b004877f 100644 --- a/src/AudioDevices.cpp +++ b/src/AudioDevices.cpp @@ -7,27 +7,9 @@ * @ref License */ -/* LICENSE - * - * Copyright (c) 2008-2021 OpenShot Studios, LLC - * . This file is part of - * OpenShot Library (libopenshot), an open-source project dedicated to - * delivering high quality video editing and animation solutions to the - * world. For more information visit . - * - * OpenShot Library (libopenshot) is free software: you can redistribute it - * and/or modify it under the terms of the GNU Lesser General Public License - * as published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * OpenShot Library (libopenshot) is distributed in the hope that it will be - * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with OpenShot Library. If not, see . - */ +// Copyright (c) 2008-2019 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later #include "AudioDevices.h" From 4a52941708a538d7684d0282521d0b4f49a3a0b4 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 3 Dec 2021 00:43:13 -0600 Subject: [PATCH 4/8] Experimental logging to detect delay on Windows playback --- src/AudioReaderSource.cpp | 5 ++++- src/Qt/PlayerPrivate.cpp | 2 ++ src/Qt/VideoCacheThread.cpp | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/AudioReaderSource.cpp b/src/AudioReaderSource.cpp index 1d0580eb..14390b24 100644 --- a/src/AudioReaderSource.cpp +++ b/src/AudioReaderSource.cpp @@ -161,6 +161,7 @@ juce::AudioBuffer* AudioReaderSource::reverse_buffer(juce::AudioBuffergetNumSamples(); int buffer_channels = buffer->getNumChannels(); @@ -224,7 +225,9 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in } // Prepare to play this audio source -void AudioReaderSource::prepareToPlay(int, double) { } +void AudioReaderSource::prepareToPlay(int, double) { + std::cout << "R: prepare" << std::endl; +} // Release all resources void AudioReaderSource::releaseResources() { } diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index e09d7415..89fb7801 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -74,6 +74,7 @@ namespace openshot || (video_position > reader->info.video_length) ) { speed = 0; + std::cout << "A: " << frame_duration.count() << std::endl; std::this_thread::sleep_for(frame_duration); continue; } @@ -132,6 +133,7 @@ namespace openshot // and shutting down, the video_frame_diff can jump to a crazy big number, and we don't // want to sleep too long (max of X seconds) if (sleep_time > sleep_time.zero() && sleep_time.count() < max_sleep_ms) { + std::cout << "B: " << sleep_time.count() << std::endl; std::this_thread::sleep_for(sleep_time); } diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 2c9d8e7b..f0b9eb52 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -115,6 +115,7 @@ namespace openshot } // Sleep for 1 frame length + std::cout << "C: " << frame_duration.count() << std::endl; std::this_thread::sleep_for(frame_duration); } From 09bcf2ab159e82bb1a157de013e761217606cdc2 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 3 Dec 2021 15:21:11 -0600 Subject: [PATCH 5/8] Fix sleep time calculation to change when the project profile is changed (big bug), and add some additional logging/debugging statements --- src/Qt/PlayerPrivate.cpp | 9 ++++++--- src/Qt/VideoCacheThread.cpp | 5 ++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 89fb7801..805de1fc 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -59,10 +59,10 @@ namespace openshot using ms = std::chrono::milliseconds; using double_ms = std::chrono::duration; - // Calculate on-screen time for a single frame in milliseconds - const auto frame_duration = double_ms(1000.0 / reader->info.fps.ToDouble()); - while (!threadShouldExit()) { + // Calculate on-screen time for a single frame in milliseconds + const auto frame_duration = double_ms(1000.0 / reader->info.fps.ToDouble()); + // Get the start time (to track how long a frame takes to render) const auto time1 = std::chrono::high_resolution_clock::now(); @@ -111,6 +111,7 @@ namespace openshot ZmqLogger::Instance()->AppendDebugMethod("PlayerPrivate::run (determine sleep)", "video_frame_diff", video_frame_diff, "video_position", video_position, "audio_position", audio_position, "speed", speed, "render_time(ms)", render_time.count(), "sleep_time(ms)", sleep_time.count()); // Adjust drift (if more than a few frames off between audio and video) + std::cout << "VD: " << video_frame_diff << std::endl; if (video_frame_diff > 0 && reader->info.has_audio && reader->info.has_video) { // Since the audio and video threads are running independently, // they will quickly get out of sync. To fix this, we calculate @@ -119,11 +120,13 @@ namespace openshot // If a frame is ahead of the audio, we sleep for longer. // If a frame is behind the audio, we sleep less (or not at all), // in order for the video to catch up. + std::cout << "D: " << duration_cast(video_frame_diff * frame_duration).count() << std::endl; sleep_time += duration_cast(video_frame_diff * frame_duration); } else if (video_frame_diff < -10 && reader->info.has_audio && reader->info.has_video) { // Skip frame(s) to catch up to the audio (if more than 10 frames behind) + std::cout << "L: " << std::fabs(video_frame_diff) / 2 << std::endl; video_position += std::fabs(video_frame_diff) / 2; // Seek forward 1/2 the difference sleep_time = sleep_time.zero(); // Don't sleep now... immediately go to next position } diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index f0b9eb52..de3b57fd 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -77,10 +77,9 @@ namespace openshot using ms = std::chrono::milliseconds; using double_ms = std::chrono::duration; - // Calculate on-screen time for a single frame in milliseconds - const auto frame_duration = double_ms(1000.0 / reader->info.fps.ToDouble()); - while (!threadShouldExit() && is_playing) { + // Calculate on-screen time for a single frame in milliseconds + const auto frame_duration = double_ms(1000.0 / reader->info.fps.ToDouble()); // Cache frames before the other threads need them // Cache frames up to the max frames. Reset to current position From f3e7208df6433f7307b15cdacf54b94ccc94b22f Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 6 Dec 2021 15:47:32 -0600 Subject: [PATCH 6/8] Adding some additional logging --- src/AudioReaderSource.cpp | 5 +---- src/Qt/PlayerPrivate.cpp | 13 +++++-------- src/Qt/VideoCacheThread.cpp | 1 - 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/AudioReaderSource.cpp b/src/AudioReaderSource.cpp index 14390b24..67359a69 100644 --- a/src/AudioReaderSource.cpp +++ b/src/AudioReaderSource.cpp @@ -161,7 +161,6 @@ juce::AudioBuffer* AudioReaderSource::reverse_buffer(juce::AudioBuffergetNumSamples(); int buffer_channels = buffer->getNumChannels(); @@ -225,9 +224,7 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in } // Prepare to play this audio source -void AudioReaderSource::prepareToPlay(int, double) { - std::cout << "R: prepare" << std::endl; -} +void AudioReaderSource::prepareToPlay(int, double) {} // Release all resources void AudioReaderSource::releaseResources() { } diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 805de1fc..6fd5f26c 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -26,7 +26,7 @@ namespace openshot , audioPlayback(new openshot::AudioPlaybackThread()) , videoPlayback(new openshot::VideoPlaybackThread(rb)) , videoCache(new openshot::VideoCacheThread()) - , speed(1), reader(NULL), last_video_position(1), max_sleep_ms(3000) + , speed(1), reader(NULL), last_video_position(1), max_sleep_ms(1500) { } // Destructor @@ -74,7 +74,6 @@ namespace openshot || (video_position > reader->info.video_length) ) { speed = 0; - std::cout << "A: " << frame_duration.count() << std::endl; std::this_thread::sleep_for(frame_duration); continue; } @@ -111,8 +110,8 @@ namespace openshot ZmqLogger::Instance()->AppendDebugMethod("PlayerPrivate::run (determine sleep)", "video_frame_diff", video_frame_diff, "video_position", video_position, "audio_position", audio_position, "speed", speed, "render_time(ms)", render_time.count(), "sleep_time(ms)", sleep_time.count()); // Adjust drift (if more than a few frames off between audio and video) - std::cout << "VD: " << video_frame_diff << std::endl; - if (video_frame_diff > 0 && reader->info.has_audio && reader->info.has_video) { + std::cout << "VD: " << video_frame_diff << " (render: " << render_time.count() << ", sleep: " << sleep_time.count() << ")" << std::endl; + if (video_frame_diff > 6 && reader->info.has_audio && reader->info.has_video) { // Since the audio and video threads are running independently, // they will quickly get out of sync. To fix this, we calculate // how far ahead or behind the video frame is, and adjust the amount @@ -120,13 +119,11 @@ namespace openshot // If a frame is ahead of the audio, we sleep for longer. // If a frame is behind the audio, we sleep less (or not at all), // in order for the video to catch up. - std::cout << "D: " << duration_cast(video_frame_diff * frame_duration).count() << std::endl; - sleep_time += duration_cast(video_frame_diff * frame_duration); + sleep_time += duration_cast((video_frame_diff / 2) * frame_duration); } - else if (video_frame_diff < -10 && reader->info.has_audio && reader->info.has_video) { + else if (video_frame_diff < -6 && reader->info.has_audio && reader->info.has_video) { // Skip frame(s) to catch up to the audio (if more than 10 frames behind) - std::cout << "L: " << std::fabs(video_frame_diff) / 2 << std::endl; video_position += std::fabs(video_frame_diff) / 2; // Seek forward 1/2 the difference sleep_time = sleep_time.zero(); // Don't sleep now... immediately go to next position } diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index de3b57fd..60d94d1d 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -114,7 +114,6 @@ namespace openshot } // Sleep for 1 frame length - std::cout << "C: " << frame_duration.count() << std::endl; std::this_thread::sleep_for(frame_duration); } From 089b30f7c50b1fc6b6a5e19d4e528f43abb1ebee Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 6 Dec 2021 17:30:45 -0600 Subject: [PATCH 7/8] Switch from milli seconds to micro seconds, for added precision. For example, instead of sleeping for 0.041666667 seconds... due to milliseconds, we were only sleeping 0.41. This would de-sync the audio and video pretty quickly, over a few seconds, causing stuttering --- src/Qt/PlayerPrivate.cpp | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 6fd5f26c..570d101f 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -26,7 +26,7 @@ namespace openshot , audioPlayback(new openshot::AudioPlaybackThread()) , videoPlayback(new openshot::VideoPlaybackThread(rb)) , videoCache(new openshot::VideoCacheThread()) - , speed(1), reader(NULL), last_video_position(1), max_sleep_ms(1500) + , speed(1), reader(NULL), last_video_position(1), max_sleep_ms(125000) { } // Destructor @@ -56,12 +56,12 @@ namespace openshot using std::chrono::duration_cast; // Types for storing time durations in whole and fractional milliseconds - using ms = std::chrono::milliseconds; - using double_ms = std::chrono::duration; + using micro_sec = std::chrono::microseconds; + using double_micro_sec = std::chrono::duration; while (!threadShouldExit()) { // Calculate on-screen time for a single frame in milliseconds - const auto frame_duration = double_ms(1000.0 / reader->info.fps.ToDouble()); + const auto frame_duration = double_micro_sec(1000000.0 / reader->info.fps.ToDouble()); // Get the start time (to track how long a frame takes to render) const auto time1 = std::chrono::high_resolution_clock::now(); @@ -101,10 +101,10 @@ namespace openshot const auto time2 = std::chrono::high_resolution_clock::now(); // Determine how many milliseconds it took to render the frame - const auto render_time = double_ms(time2 - time1); + const auto render_time = double_micro_sec(time2 - time1); // Calculate the amount of time to sleep (by subtracting the render time) - auto sleep_time = duration_cast(frame_duration - render_time); + auto sleep_time = duration_cast(frame_duration - render_time); // Debug ZmqLogger::Instance()->AppendDebugMethod("PlayerPrivate::run (determine sleep)", "video_frame_diff", video_frame_diff, "video_position", video_position, "audio_position", audio_position, "speed", speed, "render_time(ms)", render_time.count(), "sleep_time(ms)", sleep_time.count()); @@ -119,12 +119,23 @@ namespace openshot // If a frame is ahead of the audio, we sleep for longer. // If a frame is behind the audio, we sleep less (or not at all), // in order for the video to catch up. - sleep_time += duration_cast((video_frame_diff / 2) * frame_duration); + sleep_time += duration_cast((video_frame_diff / 2) * frame_duration); } - - else if (video_frame_diff < -6 && reader->info.has_audio && reader->info.has_video) { - // Skip frame(s) to catch up to the audio (if more than 10 frames behind) - video_position += std::fabs(video_frame_diff) / 2; // Seek forward 1/2 the difference + else if (video_frame_diff < -3 && reader->info.has_audio && reader->info.has_video) { + // Video frames are a bit behind, sleep less, we need to display frames more quickly + std::cout << " - sleep: " << sleep_time.count() * 0.75 << std::endl; + sleep_time = duration_cast(sleep_time * 0.75); // Sleep a little less + } + else if (video_frame_diff < -9 && reader->info.has_audio && reader->info.has_video) { + // Video frames are very behind, no sleep, we need to display frames more quickly + std::cout << " -- sleep: " << sleep_time.zero().count() << std::endl; + sleep_time = sleep_time.zero(); // Don't sleep now... immediately go to next position + } + else if (video_frame_diff < -12 && reader->info.has_audio && reader->info.has_video) { + // Video frames are very behind, jump forward the entire distance (catch up with the audio position) + // Skip frame(s) to catch up to the audio (if more than X frames behind) + std::cout << " --- sleep 0: jump += " << std::fabs(video_frame_diff) << std::endl; + video_position += std::fabs(video_frame_diff); sleep_time = sleep_time.zero(); // Don't sleep now... immediately go to next position } From 91c47eb98b34964d6a5dfada54317478f7a02836 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 7 Dec 2021 13:37:29 -0600 Subject: [PATCH 8/8] Removing debug output, and adding some improved handling of audio/video sync conditions --- src/Qt/PlayerPrivate.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 570d101f..2cd8c6db 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -110,7 +110,6 @@ namespace openshot ZmqLogger::Instance()->AppendDebugMethod("PlayerPrivate::run (determine sleep)", "video_frame_diff", video_frame_diff, "video_position", video_position, "audio_position", audio_position, "speed", speed, "render_time(ms)", render_time.count(), "sleep_time(ms)", sleep_time.count()); // Adjust drift (if more than a few frames off between audio and video) - std::cout << "VD: " << video_frame_diff << " (render: " << render_time.count() << ", sleep: " << sleep_time.count() << ")" << std::endl; if (video_frame_diff > 6 && reader->info.has_audio && reader->info.has_video) { // Since the audio and video threads are running independently, // they will quickly get out of sync. To fix this, we calculate @@ -123,18 +122,15 @@ namespace openshot } else if (video_frame_diff < -3 && reader->info.has_audio && reader->info.has_video) { // Video frames are a bit behind, sleep less, we need to display frames more quickly - std::cout << " - sleep: " << sleep_time.count() * 0.75 << std::endl; sleep_time = duration_cast(sleep_time * 0.75); // Sleep a little less } else if (video_frame_diff < -9 && reader->info.has_audio && reader->info.has_video) { // Video frames are very behind, no sleep, we need to display frames more quickly - std::cout << " -- sleep: " << sleep_time.zero().count() << std::endl; sleep_time = sleep_time.zero(); // Don't sleep now... immediately go to next position } else if (video_frame_diff < -12 && reader->info.has_audio && reader->info.has_video) { // Video frames are very behind, jump forward the entire distance (catch up with the audio position) - // Skip frame(s) to catch up to the audio (if more than X frames behind) - std::cout << " --- sleep 0: jump += " << std::fabs(video_frame_diff) << std::endl; + // Skip frame(s) to catch up to the audio video_position += std::fabs(video_frame_diff); sleep_time = sleep_time.zero(); // Don't sleep now... immediately go to next position } @@ -144,7 +140,6 @@ namespace openshot // and shutting down, the video_frame_diff can jump to a crazy big number, and we don't // want to sleep too long (max of X seconds) if (sleep_time > sleep_time.zero() && sleep_time.count() < max_sleep_ms) { - std::cout << "B: " << sleep_time.count() << std::endl; std::this_thread::sleep_for(sleep_time); }