From 133bae40c3be7e1b531f9a96d5186fa211d98ca2 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 26 Jan 2022 17:56:33 -0600 Subject: [PATCH] Adding pre-roll to VideoCacheThread, and making video & audio threads wait for the isReady() method before playing. Also fixed an audio bug where our internal buffer was not being cleared on seek. Removed some unused caching from Clip, and did some minor refactor on FrameMapper/Clip cache clearing. --- src/AudioReaderSource.cpp | 8 +++-- src/AudioReaderSource.h | 5 +++ src/Clip.cpp | 3 -- src/ClipBase.h | 2 -- src/FrameMapper.cpp | 34 +++++++++---------- src/Qt/AudioPlaybackThread.cpp | 13 +++----- src/Qt/AudioPlaybackThread.h | 11 +++--- src/Qt/PlayerPrivate.cpp | 61 ++++++++++++++++++---------------- src/Qt/PlayerPrivate.h | 5 ++- src/Qt/VideoCacheThread.cpp | 50 +++++++++++++++++++++++----- src/Qt/VideoCacheThread.h | 16 +++++++-- src/QtPlayer.cpp | 2 +- src/Timeline.cpp | 14 +++----- 13 files changed, 137 insertions(+), 87 deletions(-) diff --git a/src/AudioReaderSource.cpp b/src/AudioReaderSource.cpp index 67359a69..a9fb7aa2 100644 --- a/src/AudioReaderSource.cpp +++ b/src/AudioReaderSource.cpp @@ -20,7 +20,7 @@ using namespace openshot; // Constructor that reads samples from a reader AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number, int buffer_size) - : reader(audio_reader), frame_number(starting_frame_number), + : reader(audio_reader), frame_number(starting_frame_number), videoCache(NULL), size(buffer_size), position(0), frame_position(0), estimated_frame(0), speed(1) { // Initialize an audio buffer (based on reader) @@ -168,7 +168,7 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in int number_to_copy = 0; // Do we need more samples? - if (speed == 1) { + if (speed == 1 && videoCache->isReady()) { // Only refill buffers if speed is normal if ((reader && reader->IsOpen() && !frame) or (reader && reader->IsOpen() && buffer_samples - position < info.numSamples)) @@ -177,6 +177,10 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in } else { // Fill buffer with silence and clear current frame info.buffer->clear(); + + // Empty internal buffer also + buffer->clear(); + position = 0; return; } diff --git a/src/AudioReaderSource.h b/src/AudioReaderSource.h index 32356567..81301f4e 100644 --- a/src/AudioReaderSource.h +++ b/src/AudioReaderSource.h @@ -14,6 +14,7 @@ #define OPENSHOT_AUDIOREADERSOURCE_H #include "ReaderBase.h" +#include "Qt/VideoCacheThread.h" #include #include @@ -42,6 +43,7 @@ namespace openshot int64_t frame_position; /// The position of the current frame's buffer double estimated_frame; /// The estimated frame position of the currently playing buffer int estimated_samples_per_frame; /// The estimated samples per frame of video + openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking) /// Get more samples from the reader void GetMoreSamplesFromReader(); @@ -103,6 +105,9 @@ namespace openshot /// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) int getSpeed() const { return speed; } + /// Set playback video cache thread (for pre-roll reference) + void setVideoCache(openshot::VideoCacheThread *newCache) { videoCache = newCache; } + /// Set Reader void Reader(ReaderBase *audio_reader) { reader = audio_reader; } /// Get Reader diff --git a/src/Clip.cpp b/src/Clip.cpp index 7fdd9555..b323f657 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -1198,9 +1198,6 @@ void Clip::AddEffect(EffectBase* effect) } } #endif - - // Clear cache - cache.Clear(); } // Remove an effect from the clip diff --git a/src/ClipBase.h b/src/ClipBase.h index 83054311..9ac5cb44 100644 --- a/src/ClipBase.h +++ b/src/ClipBase.h @@ -47,8 +47,6 @@ namespace openshot { Json::Value add_property_choice_json(std::string name, int value, int selected_value) const; public: - CacheMemory cache; - /// Constructor for the base clip ClipBase() : position(0.0), diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index c405bf26..74666755 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -655,24 +655,24 @@ void FrameMapper::Close() // Close internal reader reader->Close(); - - // Clear the fields & frames lists - fields.clear(); - frames.clear(); - - // Mark as dirty - is_dirty = true; - - // Clear cache - final_cache.Clear(); - - // Deallocate resample buffer - if (avr) { - SWR_CLOSE(avr); - SWR_FREE(&avr); - avr = NULL; - } } + + // Clear the fields & frames lists + fields.clear(); + frames.clear(); + + // Mark as dirty + is_dirty = true; + + // Clear cache + final_cache.Clear(); + + // Deallocate resample buffer + if (avr) { + SWR_CLOSE(avr); + SWR_FREE(&avr); + avr = NULL; + } } diff --git a/src/Qt/AudioPlaybackThread.cpp b/src/Qt/AudioPlaybackThread.cpp index b13ce5a3..d1654458 100644 --- a/src/Qt/AudioPlaybackThread.cpp +++ b/src/Qt/AudioPlaybackThread.cpp @@ -92,7 +92,7 @@ namespace openshot } // Constructor - AudioPlaybackThread::AudioPlaybackThread() + AudioPlaybackThread::AudioPlaybackThread(openshot::VideoCacheThread* cache) : juce::Thread("audio-playback") , player() , transport() @@ -103,6 +103,7 @@ namespace openshot , buffer_size(7000) , is_playing(false) , time_thread("audio-buffer") + , videoCache(cache) { } @@ -125,8 +126,8 @@ namespace openshot sampleRate = reader->info.sample_rate; numChannels = reader->info.channels; - // TODO: Update transport or audio source's sample rate, incase the sample rate - // is different than the original Reader + // Set video cache thread + source->setVideoCache(videoCache); // Mark as 'playing' Play(); @@ -139,12 +140,6 @@ namespace openshot return std::shared_ptr(); } - // Get the currently playing frame number - int64_t AudioPlaybackThread::getCurrentFramePosition() - { - return source ? source->getEstimatedFrame() : 0; - } - // Seek the audio thread void AudioPlaybackThread::Seek(int64_t new_position) { diff --git a/src/Qt/AudioPlaybackThread.h b/src/Qt/AudioPlaybackThread.h index ac9819da..821b7138 100644 --- a/src/Qt/AudioPlaybackThread.h +++ b/src/Qt/AudioPlaybackThread.h @@ -19,6 +19,7 @@ #include "AudioReaderSource.h" #include "AudioDevices.h" #include "AudioReaderSource.h" +#include "Qt/VideoCacheThread.h" #include #include @@ -75,20 +76,22 @@ public: int buffer_size; bool is_playing; juce::TimeSliceThread time_thread; + openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking) + /// Constructor - AudioPlaybackThread(); + AudioPlaybackThread(openshot::VideoCacheThread* cache); /// Destructor ~AudioPlaybackThread(); /// Set the current thread's reader void Reader(openshot::ReaderBase *reader); + /// Return the current audio transport buffer size (to determine latency) + int getBufferSize() { return buffer_size; } + /// Get the current frame object (which is filling the buffer) std::shared_ptr getFrame(); - /// Get the current frame number being played - int64_t getCurrentFramePosition(); - /// Play the audio void Play(); diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 969200b8..cf63c8e0 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -17,19 +17,20 @@ #include #include // for std::this_thread::sleep_for -#include // for std::chrono milliseconds, high_resolution_clock +#include // for std::chrono microseconds, high_resolution_clock namespace openshot { int close_to_sync = 5; // Constructor PlayerPrivate::PlayerPrivate(openshot::RendererBase *rb) - : renderer(rb), Thread("player"), video_position(1), audio_position(0) - , audioPlayback(new openshot::AudioPlaybackThread()) - , videoPlayback(new openshot::VideoPlaybackThread(rb)) - , videoCache(new openshot::VideoCacheThread()) - , speed(1), reader(NULL), last_video_position(1), max_sleep_ms(125000) - { } + : renderer(rb), Thread("player"), video_position(1), audio_position(0), + speed(1), reader(NULL), last_video_position(1), max_sleep_ms(125000), playback_frames(0) + { + videoCache = new openshot::VideoCacheThread(); + audioPlayback = new openshot::AudioPlaybackThread(videoCache); + videoPlayback = new openshot::VideoPlaybackThread(rb); + } // Destructor PlayerPrivate::~PlayerPrivate() @@ -61,26 +62,34 @@ namespace openshot using micro_sec = std::chrono::microseconds; using double_micro_sec = std::chrono::duration; - // Time-based video sync - auto start_time = std::chrono::high_resolution_clock::now(); ///< timestamp playback starts + // Calculate latency of audio thread (i.e. how many microseconds before samples are audible) + // TODO: This is the experimental amount of latency I have on my system audio playback + //const auto audio_latency = double_micro_sec(1000000.0 * (audioPlayback->getBufferSize() / reader->info.sample_rate)); + const auto audio_latency = double_micro_sec(240000); + + // Init start_time of playback + std::chrono::time_point start_time; + start_time = std::chrono::time_point_cast(std::chrono::high_resolution_clock::now()); ///< timestamp playback starts while (!threadShouldExit()) { // Calculate on-screen time for a single frame const auto frame_duration = double_micro_sec(1000000.0 / reader->info.fps.ToDouble()); const auto max_sleep = frame_duration * 4; ///< Don't sleep longer than X times a frame duration - // Get the current video frame (if it's different) + // Get the current video frame frame = getFrame(); // Pausing Code (if frame has not changed) - if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length)) + if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length) || !videoCache->isReady()) { - // Set start time to prepare for next playback period and reset frame counter - start_time = std::chrono::high_resolution_clock::now(); - playback_frames = 0; - // Sleep for a fraction of frame duration std::this_thread::sleep_for(frame_duration / 4); + audioPlayback->Seek(video_position); + + // Reset current playback start time + start_time = std::chrono::time_point_cast(std::chrono::high_resolution_clock::now()); + playback_frames = 0; + continue; } @@ -91,21 +100,10 @@ namespace openshot // Keep track of the last displayed frame last_video_position = video_position; - // How many frames ahead or behind is the video thread? - // Only calculate this if a reader contains both an audio and video thread - int64_t video_frame_diff = 0; - if (reader->info.has_audio && reader->info.has_video) { - audio_position = audioPlayback->getCurrentFramePosition(); - video_frame_diff = video_position - audio_position; - // Seek to audio frame again (since we are not in normal speed, and not paused) - if (speed != 1) { - audioPlayback->Seek(video_position); - } - } - // Calculate the diff between 'now' and the predicted frame end time const auto current_time = std::chrono::high_resolution_clock::now(); - const auto remaining_time = double_micro_sec(start_time + (frame_duration * playback_frames) - current_time); + const auto remaining_time = double_micro_sec(start_time + audio_latency + + (frame_duration * playback_frames) - current_time); // Sleep to display video image on screen if (remaining_time > remaining_time.zero() ) { @@ -151,6 +149,13 @@ namespace openshot return std::shared_ptr(); } + // Seek to a new position + void PlayerPrivate::Seek(int64_t new_position) + { + video_position = new_position; + last_video_position = 0; + } + // Start video/audio playback bool PlayerPrivate::startPlayback() { diff --git a/src/Qt/PlayerPrivate.h b/src/Qt/PlayerPrivate.h index a2a8644c..107341d6 100644 --- a/src/Qt/PlayerPrivate.h +++ b/src/Qt/PlayerPrivate.h @@ -30,7 +30,7 @@ namespace openshot class PlayerPrivate : juce::Thread { std::shared_ptr frame; /// The current frame - int64_t playback_frames = 0; /// The # of frames since playback started + int64_t playback_frames; /// The # of frames since playback started int64_t video_position; /// The current frame position. int64_t audio_position; /// The current frame position. openshot::ReaderBase *reader; /// The reader which powers this player @@ -53,6 +53,9 @@ namespace openshot /// Start the video/audio playback bool startPlayback(); + /// Seek to a new frame # + void Seek(int64_t new_position); + /// Stop the video/audio playback void stopPlayback(); diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 4edf7a18..f1ddf42a 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -27,7 +27,8 @@ namespace openshot // Constructor VideoCacheThread::VideoCacheThread() : Thread("video-cache"), speed(1), is_playing(false), - reader(NULL), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 2), current_display_frame(1) + reader(NULL), current_display_frame(1), cached_frame_count(0), + min_frames_ahead(12), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6) { } @@ -39,9 +40,18 @@ namespace openshot // Seek the reader to a particular frame number void VideoCacheThread::Seek(int64_t new_position) { - current_display_frame = new_position; + requested_display_frame = new_position; } + // Seek the reader to a particular frame number and optionally start the pre-roll + void VideoCacheThread::Seek(int64_t new_position, bool start_preroll) + { + if (start_preroll && reader && reader->GetCache() && !reader->GetCache()->Contains(new_position)) { + cached_frame_count = 0; + } + Seek(new_position); + } + // Play the video void VideoCacheThread::Play() { // Start playing @@ -54,6 +64,11 @@ namespace openshot is_playing = false; } + // Is cache ready for playback (pre-roll) + bool VideoCacheThread::isReady() { + return (cached_frame_count > min_frames_ahead); + } + // Start the thread void VideoCacheThread::run() { @@ -73,13 +88,18 @@ namespace openshot bytes_per_frame = last_cached_frame->GetBytes(); } - // Calculate # of frames on Timeline cache + // Calculate # of frames on Timeline cache (when paused) if (reader->GetCache() && reader->GetCache()->GetMaxBytes() > 0) { - // Use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it) - max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2; - if (max_frames_ahead > 1000) { - // Ignore values that are too large, and default to a safer value - max_frames_ahead = OPEN_MP_NUM_PROCESSORS * 2; + if (speed == 0) { + // When paused, use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it) + max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2; + if (max_frames_ahead > 300) { + // Ignore values that are too large, and default to a safer value + max_frames_ahead = 300; + } + } else { + // When playing back video (speed == 1), keep cache # small + max_frames_ahead = min_frames_ahead; } } @@ -100,7 +120,9 @@ namespace openshot ending_frame = starting_frame - max_frames_ahead; } + // Loop through range of frames (and cache them) for (int64_t cache_frame = starting_frame; cache_frame != ending_frame; cache_frame += increment) { + cached_frame_count++; if (reader && reader->GetCache() && !reader->GetCache()->Contains(cache_frame)) { try { @@ -110,8 +132,20 @@ namespace openshot } catch (const OutOfBoundsFrame & e) { } } + // Check if the user has seeked outside the cache range + if (requested_display_frame != current_display_frame) { + // cache will restart at a new position + if (speed >= 0 && (requested_display_frame < starting_frame || requested_display_frame > ending_frame)) { + break; + } else if (speed < 0 && (requested_display_frame > starting_frame || requested_display_frame < ending_frame)) { + break; + } + } } + // Update current display frame + current_display_frame = requested_display_frame; + // Sleep for a fraction of frame duration std::this_thread::sleep_for(frame_duration / 4); } diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index cc120b44..dd21bce5 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -32,10 +32,14 @@ namespace openshot std::shared_ptr last_cached_frame; int speed; bool is_playing; + int64_t requested_display_frame; int64_t current_display_frame; + int64_t cached_frame_count = 0; ReaderBase *reader; + int min_frames_ahead; int max_frames_ahead; + /// Constructor VideoCacheThread(); /// Destructor @@ -47,8 +51,11 @@ namespace openshot /// Play the video void Play(); - /// Seek the reader to a particular frame number - void Seek(int64_t new_position); + /// Seek the reader to a particular frame number + void Seek(int64_t new_position); + + /// Seek the reader to a particular frame number and optionally start the pre-roll + void Seek(int64_t new_position, bool start_preroll); /// Set Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) void setSpeed(int new_speed) { speed = new_speed; } @@ -65,8 +72,11 @@ namespace openshot /// Parent class of VideoCacheThread friend class PlayerPrivate; friend class QtPlayer; - }; + public: + /// Is cache ready for video/audio playback + bool isReady(); + }; } #endif // OPENSHOT_VIDEO_CACHE_THREAD_H diff --git a/src/QtPlayer.cpp b/src/QtPlayer.cpp index 13e07c77..9f148572 100644 --- a/src/QtPlayer.cpp +++ b/src/QtPlayer.cpp @@ -155,7 +155,7 @@ namespace openshot // Check for seek if (reader && threads_started && new_frame > 0) { // Notify cache thread that seek has occurred - p->videoCache->Seek(new_frame); + p->videoCache->Seek(new_frame, true); // Notify audio thread that seek has occurred p->audioPlayback->Seek(new_frame); diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 2c95a90c..97d6c221 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -343,7 +343,6 @@ void Timeline::AddClip(Clip* clip) clip->ParentTimeline(this); // Clear cache of clip and nested reader (if any) - clip->cache.Clear(); if (clip->Reader() && clip->Reader()->GetCache()) clip->Reader()->GetCache()->Clear(); @@ -734,9 +733,8 @@ void Timeline::Close() // Mark timeline as closed is_open = false; - // Clear cache - if (final_cache) - final_cache->Clear(); + // Clear all cache + ClearAllCache(); } // Open the reader (and start consuming resources) @@ -754,7 +752,6 @@ bool Timeline::isEqual(double a, double b) // Get an openshot::Frame object for a specific frame number of this reader. std::shared_ptr Timeline::GetFrame(int64_t requested_frame) { - // Adjust out of bounds frame number if (requested_frame < 1) requested_frame = 1; @@ -789,9 +786,6 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) return frame; } - // Minimum number of frames to process (for performance reasons) - int minimum_frames = OPEN_MP_NUM_PROCESSORS; - // Get a list of clips that intersect with the requested section of timeline // This also opens the readers for intersecting clips, and marks non-intersecting clips as 'needs closing' std::vector nearby_clips; @@ -1474,7 +1468,9 @@ void Timeline::ClearAllCache() { const std::lock_guard lock(getFrameMutex); // Clear primary cache - final_cache->Clear(); + if (final_cache) { + final_cache->Clear(); + } // Loop through all clips for (auto clip : clips)