From a5f35fb42a842bcd88b838ff611335b2f0c6dd68 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 14 Jan 2022 15:16:04 -0600 Subject: [PATCH] Refactoring the VideoCacheThread to check every frame before requesting it. Adding a new method: Contains() to our cache objects, to facilitate this. Removing cache clearning experimental code from Timeline (causing playback issues). Refactoring PrivatePlayer playback timing code, to calculate an average # of frame difference between audio and video threads, and slowly adjust back towards zero when needed. --- src/CacheBase.h | 4 ++ src/CacheDisk.cpp | 9 ++++ src/CacheDisk.h | 4 ++ src/CacheMemory.cpp | 9 ++++ src/CacheMemory.h | 4 ++ src/Qt/PlayerPrivate.cpp | 78 ++++++++++++++++++++++++------- src/Qt/VideoCacheThread.cpp | 93 ++++++++++++++++++------------------- src/Qt/VideoCacheThread.h | 13 +----- src/Timeline.cpp | 9 ---- 9 files changed, 139 insertions(+), 84 deletions(-) diff --git a/src/CacheBase.h b/src/CacheBase.h index 8ed89935..4377a292 100644 --- a/src/CacheBase.h +++ b/src/CacheBase.h @@ -53,6 +53,10 @@ namespace openshot { /// Clear the cache of all frames virtual void Clear() = 0; + /// @brief Check if frame is already contained in cache + /// @param frame_number The frame number to be checked + virtual bool Contains(int64_t frame_number) = 0; + /// Count the frames in the queue virtual int64_t Count() = 0; diff --git a/src/CacheDisk.cpp b/src/CacheDisk.cpp index bf66bcd9..8cc00003 100644 --- a/src/CacheDisk.cpp +++ b/src/CacheDisk.cpp @@ -202,6 +202,15 @@ void CacheDisk::Add(std::shared_ptr frame) } } +// Check if frame is already contained in cache +bool CacheDisk::Contains(int64_t frame_number) { + if (frames.count(frame_number) > 0) { + return true; + } else { + return false; + } +} + // Get a frame from the cache (or NULL shared_ptr if no frame is found) std::shared_ptr CacheDisk::GetFrame(int64_t frame_number) { diff --git a/src/CacheDisk.h b/src/CacheDisk.h index 6c0d1b77..7611728c 100644 --- a/src/CacheDisk.h +++ b/src/CacheDisk.h @@ -82,6 +82,10 @@ namespace openshot { /// Clear the cache of all frames void Clear(); + /// @brief Check if frame is already contained in cache + /// @param frame_number The frame number to be checked + bool Contains(int64_t frame_number); + /// Count the frames in the queue int64_t Count(); diff --git a/src/CacheMemory.cpp b/src/CacheMemory.cpp index 73c20f89..780d59b1 100644 --- a/src/CacheMemory.cpp +++ b/src/CacheMemory.cpp @@ -129,6 +129,15 @@ void CacheMemory::Add(std::shared_ptr frame) } } +// Check if frame is already contained in cache +bool CacheMemory::Contains(int64_t frame_number) { + if (frames.count(frame_number) > 0) { + return true; + } else { + return false; + } +} + // Get a frame from the cache (or NULL shared_ptr if no frame is found) std::shared_ptr CacheMemory::GetFrame(int64_t frame_number) { diff --git a/src/CacheMemory.h b/src/CacheMemory.h index 9be0fdcf..7e5913f0 100644 --- a/src/CacheMemory.h +++ b/src/CacheMemory.h @@ -65,6 +65,10 @@ namespace openshot { /// Clear the cache of all frames void Clear(); + /// @brief Check if frame is already contained in cache + /// @param frame_number The frame number to be checked + bool Contains(int64_t frame_number); + /// Count the frames in the queue int64_t Count(); diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index c707f723..ee98ffc0 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -15,6 +15,7 @@ #include "Exceptions.h" #include "ZmqLogger.h" +#include #include // for std::this_thread::sleep_for #include // for std::chrono milliseconds, high_resolution_clock @@ -59,22 +60,38 @@ namespace openshot // Types for storing time durations in whole and fractional microseconds using micro_sec = std::chrono::microseconds; using double_micro_sec = std::chrono::duration; + auto sleep_adjustment = double_micro_sec(0.0); + std::deque video_diff_history; + + // Init sleep adjustment code + float average_video_diff = 0; ///< Average number of frames difference between audio and video thread + bool enable_adjustment = false; ///< Toggle desync adjustments ON & OFF (i.e. only adjust when needed) + float video_diff_threshold = 1.0; ///< Number of frames difference to trigger an adjustment while (!threadShouldExit()) { // Get the start time (to track how long a frame takes to render) const auto time1 = std::chrono::high_resolution_clock::now(); + // Calculate on-screen time for a single frame in microseconds const auto frame_duration = double_micro_sec(1000000.0 / reader->info.fps.ToDouble()); - const auto max_sleep = frame_duration * 1.5; + const auto max_sleep = frame_duration * 4; ///< Don't sleep longer than X times a frame duration + const auto sleep_adjustment_increment = frame_duration / 100000; ///< Tiny fraction of frame duration + + // Adjust size used in averaging desync calculation + auto history_size = reader->info.fps.ToInt(); ///< How many values should we average together (i.e. use FPS) // Get the current video frame (if it's different) frame = getFrame(); // Experimental Pausing Code (if frame has not changed) - if ((speed == 0 && video_position == last_video_position) - || (video_position > reader->info.video_length) - ) { + if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length)) + { speed = 0; + // Clear sleep adjustments & average diff + sleep_adjustment = double_micro_sec(0.0); + average_video_diff = 0.0; + video_diff_history.clear(); + // Sleep for 1 frame std::this_thread::sleep_for(frame_duration); continue; } @@ -87,15 +104,48 @@ namespace openshot last_video_position = video_position; // How many frames ahead or behind is the video thread? + // Only calculate this if a reader contains both an audio and video thread int64_t video_frame_diff = 0; if (reader->info.has_audio && reader->info.has_video) { - if (speed != 1) - // Set audio frame again (since we are not in normal speed, and not paused) - audioPlayback->Seek(video_position); - - // Only calculate this if a reader contains both an audio and video thread audio_position = audioPlayback->getCurrentFramePosition(); video_frame_diff = video_position - audio_position; + // Seek to audio frame again (since we are not in normal speed, and not paused) + if (speed != 1) { + audioPlayback->Seek(video_position); + } + } + + // Add video diff to history (for averaging sync delays over time) + video_diff_history.push_front(video_frame_diff); + if (video_diff_history.size() > history_size) { + // Remove oldest element (keep queue frozen with only most recent diffs) + video_diff_history.pop_back(); + // Loop through recent video diff values + int64_t diff_sum = 0; + for(auto it=video_diff_history.begin(); it!=video_diff_history.end();++it) { + diff_sum += *it; + } + // Calculate average video diff (to be used for syncing video to audio) + average_video_diff = float(diff_sum) / history_size; + } + + // Should we enable the adjustment? (i.e. too much desync) + if (!enable_adjustment && fabs(average_video_diff) > video_diff_threshold) { + enable_adjustment = true; + } else if (enable_adjustment && fabs(average_video_diff) > 0.0 && fabs(average_video_diff) < 0.25) { + // Close to zero, stop adjusting + enable_adjustment = false; + } + + // Calculate sync adjustment (if adjustment is triggered) + if (enable_adjustment) { + if (average_video_diff < 0.0) { + // We need to slowly decrease the sleep + sleep_adjustment -= sleep_adjustment_increment; + } else if (video_frame_diff > 0.0) { + // We need to slowly increase the sleep + sleep_adjustment += sleep_adjustment_increment; + } } // Get the end time (to track how long a frame takes to render) @@ -105,13 +155,7 @@ namespace openshot const auto render_time = double_micro_sec(time2 - time1); // Calculate the amount of time to sleep (by subtracting the render time) - auto sleep_time = duration_cast(frame_duration - render_time); - // Raising video_frame_diff to the power of 3 - // Preserves the sign (+/-) and corrects more aggressively depending on how - // out of sync the video is. - sleep_time = sleep_time + std::chrono::microseconds((long(powf(100 * video_frame_diff, 3) ))); - - ZmqLogger::Instance()->AppendDebugMethod("PlayerPrivate::run (determine sleep)", "video_frame_diff", video_frame_diff, "video_position", video_position, "audio_position", audio_position, "speed", speed, "render_time(ms)", render_time.count(), "sleep_time(ms)", sleep_time.count()); + auto sleep_time = duration_cast(frame_duration - render_time + sleep_adjustment); // Sleep if sleep_time is greater than zero after correction // (leaving the video frame on the screen for the correct amount of time) @@ -140,7 +184,7 @@ namespace openshot else { // Update cache on which frame was retrieved - videoCache->setCurrentFramePosition(video_position); + videoCache->Seek(video_position); // return frame from reader return reader->GetFrame(video_position); diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 60d94d1d..77f8a78f 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -26,8 +26,8 @@ namespace openshot { // Constructor VideoCacheThread::VideoCacheThread() - : Thread("video-cache"), speed(1), is_playing(false), position(1) - , reader(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS * 4), current_display_frame(1) + : Thread("video-cache"), speed(1), is_playing(false), + reader(NULL), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 2), current_display_frame(1) { } @@ -36,25 +36,10 @@ namespace openshot { } - // Get the currently playing frame number (if any) - int64_t VideoCacheThread::getCurrentFramePosition() - { - if (frame) - return frame->number; - else - return 0; - } - - // Set the currently playing frame number (if any) - void VideoCacheThread::setCurrentFramePosition(int64_t current_frame_number) - { - current_display_frame = current_frame_number; - } - // Seek the reader to a particular frame number void VideoCacheThread::Seek(int64_t new_position) { - position = new_position; + current_display_frame = new_position; } // Play the video @@ -73,7 +58,6 @@ namespace openshot void VideoCacheThread::run() { // Types for storing time durations in whole and fractional milliseconds - std::shared_ptr smallest_frame = NULL; using ms = std::chrono::milliseconds; using double_ms = std::chrono::duration; @@ -81,37 +65,52 @@ namespace openshot // Calculate on-screen time for a single frame in milliseconds const auto frame_duration = double_ms(1000.0 / reader->info.fps.ToDouble()); - // Cache frames before the other threads need them - // Cache frames up to the max frames. Reset to current position - // if cache gets too far away from display frame. Cache frames - // even when player is paused (i.e. speed 0). - while (((position - current_display_frame) < max_concurrent_frames) && is_playing) - { - // Only cache up till the max_concurrent_frames amount... then sleep - try - { - if (reader) { - ZmqLogger::Instance()->AppendDebugMethod("VideoCacheThread::run (cache frame)", "position", position, "current_display_frame", current_display_frame, "max_concurrent_frames", max_concurrent_frames, "needed_frames", (position - current_display_frame)); + // Calculate bytes per frame. If we have a reference openshot::Frame, use that instead (the preview + // window can be smaller, can thus reduce the bytes per frame) + int64_t bytes_per_frame = (reader->info.height * reader->info.width * 4) + + (reader->info.sample_rate * reader->info.channels * 4); + if (last_cached_frame && last_cached_frame->has_image_data && last_cached_frame->has_audio_data) { + bytes_per_frame = last_cached_frame->GetBytes(); + } - // Force the frame to be generated - smallest_frame = reader->GetCache()->GetSmallestFrame(); - if (smallest_frame && smallest_frame->number > current_display_frame) { - // Cache position has gotten too far away from current display frame. - // Reset the position to the current display frame. - position = current_display_frame; - } - reader->GetFrame(position); - } + // Calculate # of frames on Timeline cache + if (reader->GetCache() && reader->GetCache()->GetMaxBytes() > 0) { + // Use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it) + max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2; + if (max_frames_ahead > 1000) { + // Ignore values that are too large, and default to a safer value + max_frames_ahead = OPEN_MP_NUM_PROCESSORS * 2; + } + } - } - catch (const OutOfBoundsFrame & e) - { - // Ignore out of bounds frame exceptions - } + // Calculate increment (based on speed) + // Support caching in both directions + int16_t increment = 1; + if (speed < 0) { + increment = -1; + } - // Increment frame number - position++; - } + // Always cache frames from the current display position to our maximum (based on the cache size). + // Frames which are already cached are basically free. Only uncached frames have a big CPU cost. + // By always looping through the expected frame range, we can fill-in missing frames caused by a + // fragmented cache object (i.e. the user clicking all over the timeline). + int64_t starting_frame = current_display_frame; + int64_t ending_frame = starting_frame + max_frames_ahead; + if (speed < 0) { + ending_frame = starting_frame - max_frames_ahead; + } + + for (int64_t cache_frame = starting_frame; cache_frame != ending_frame; cache_frame += increment) { + if (reader && reader->GetCache() && !reader->GetCache()->Contains(cache_frame)) { + try + { + // This frame is not already cached... so request it again (to force the creation & caching) + // This will also re-order the missing frame to the front of the cache + last_cached_frame = reader->GetFrame(cache_frame); + } + catch (const OutOfBoundsFrame & e) { } + } + } // Sleep for 1 frame length std::this_thread::sleep_for(frame_duration); diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index 7ef1da8b..cc120b44 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -28,25 +28,19 @@ namespace openshot */ class VideoCacheThread : Thread { - private: - std::atomic_int position; - protected: - std::shared_ptr frame; + std::shared_ptr last_cached_frame; int speed; bool is_playing; int64_t current_display_frame; ReaderBase *reader; - int max_concurrent_frames; + int max_frames_ahead; /// Constructor VideoCacheThread(); /// Destructor ~VideoCacheThread(); - /// Get the currently playing frame number (if any) - int64_t getCurrentFramePosition(); - /// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) int getSpeed() const { return speed; } @@ -56,9 +50,6 @@ namespace openshot /// Seek the reader to a particular frame number void Seek(int64_t new_position); - /// Set the currently displaying frame number - void setCurrentFramePosition(int64_t current_frame_number); - /// Set Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) void setSpeed(int new_speed) { speed = new_speed; } diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 0de6045f..2c95a90c 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -789,15 +789,6 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) return frame; } - // Check if previous frame was cached? (if not, assume we are seeking somewhere else on the Timeline, and need - // to clear all cache (for continuity sake). For example, jumping back to a previous spot can cause issues with audio - // data where the new jump location doesn't match up with the previously cached audio data. - std::shared_ptr previous_frame = final_cache->GetFrame(requested_frame - 1); - if (!previous_frame) { - // Seeking to new place on timeline (destroy cache) - ClearAllCache(); - } - // Minimum number of frames to process (for performance reasons) int minimum_frames = OPEN_MP_NUM_PROCESSORS;