From 35d4629df64185005b6cfb7f66a2e4010cbc3a5e Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 9 Feb 2022 17:29:04 -0600 Subject: [PATCH] Large refactor to caching and playback timing: - Support different speeds (-1X, 2X, 4X, etc...) - Clamp getFrame between 1 and timeline length - Support rewind using new timing code - Caching in both directions (based on previous speed) - Removing mutex from Timeline::GetFrame (cached path) - Caching supports actual speed now - Bust cache and trigger pre-roll if we encounter too many uncached frames. This will re-sync the video + audio threads. - Sleep less in cache loop / Adjusting min frames to 24 (trial and error). Too few and backwards playback suffers. Too many and all playback waits. --- src/Qt/PlayerPrivate.cpp | 38 ++++++++++++++++++++++++++------- src/Qt/PlayerPrivate.h | 1 + src/Qt/VideoCacheThread.cpp | 42 ++++++++++++++++++++++++++++++------- src/Qt/VideoCacheThread.h | 5 +++-- src/Timeline.cpp | 17 +-------------- 5 files changed, 71 insertions(+), 32 deletions(-) diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 7961d1f9..88f4771e 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -13,7 +13,6 @@ #include "PlayerPrivate.h" #include "Exceptions.h" -#include "ZmqLogger.h" #include #include // for std::this_thread::sleep_for @@ -68,15 +67,16 @@ namespace openshot while (!threadShouldExit()) { // Calculate on-screen time for a single frame - const auto frame_duration = double_micro_sec(1000000.0 / reader->info.fps.ToDouble()); + int frame_speed = std::max(abs(speed), 1); + const auto frame_duration = double_micro_sec(1000000.0 / (reader->info.fps.ToDouble() * frame_speed)); const auto max_sleep = frame_duration * 4; ///< Don't sleep longer than X times a frame duration // Pausing Code (if frame has not changed) // Also pause at end of timeline, and pause if 'playing' and // the pre-roll is not yet ready if ((speed == 0 && video_position == last_video_position) || - (video_position > reader->info.video_length) || - (speed == 1 && !videoCache->isReady())) + (speed != 0 && last_speed != speed) || + !videoCache->isReady()) { // Sleep for a fraction of frame duration std::this_thread::sleep_for(frame_duration / 4); @@ -84,6 +84,10 @@ namespace openshot // Reset current playback start time start_time = std::chrono::time_point_cast(std::chrono::system_clock::now()); playback_frames = 0; + last_speed = speed; + + // Seek audio thread (since audio is also paused) + audioPlayback->Seek(video_position); continue; } @@ -97,6 +101,7 @@ namespace openshot // Keep track of the last displayed frame last_video_position = video_position; + last_speed = speed; // Calculate the diff between 'now' and the predicted frame end time const auto current_time = std::chrono::system_clock::now(); @@ -111,6 +116,15 @@ namespace openshot // Protect against invalid or too-long sleep times std::this_thread::sleep_for(max_sleep); } + } else { + // DEBUG: DELETE THIS SOON + // Video position is running too far behind (negative sleep duration) + auto num_frames_to_advance = roundToInt(fabs(double_micro_sec(remaining_time / frame_duration).count())); + if (speed > 0 && num_frames_to_advance > 1) { + std::cout << "frames behind ++: " << num_frames_to_advance << ", playback_frames: " << playback_frames << std::endl; + } else if (speed < 0 && num_frames_to_advance > 1) { + std::cout << "frames behind --: " << num_frames_to_advance << ", playback_frames: " << playback_frames << std::endl; + } } } } @@ -120,17 +134,27 @@ namespace openshot { try { // Get the next frame (based on speed) - if (video_position + speed >= 1 && video_position + speed <= reader->info.video_length) + if (video_position + speed >= 1 && video_position + speed <= reader->info.video_length) { video_position = video_position + speed; + } else if (video_position + speed < 1) { + // Start of reader (prevent negative frame number and pause playback) + video_position = 1; + speed = 0; + } else if (video_position + speed > reader->info.video_length) { + // End of reader (prevent negative frame number and pause playback) + video_position = reader->info.video_length; + speed = 0; + } + if (frame && frame->number == video_position && video_position == last_video_position) { // return cached frame return frame; } else { - // Increment playback frames - playback_frames += speed; + // Increment playback frames (always in the positive direction) + playback_frames += std::abs(speed); // Update cache on which frame was retrieved videoCache->Seek(video_position); diff --git a/src/Qt/PlayerPrivate.h b/src/Qt/PlayerPrivate.h index 107341d6..f9137821 100644 --- a/src/Qt/PlayerPrivate.h +++ b/src/Qt/PlayerPrivate.h @@ -38,6 +38,7 @@ namespace openshot openshot::VideoPlaybackThread *videoPlayback; /// The video thread openshot::VideoCacheThread *videoCache; /// The cache thread int speed; /// The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) + int last_speed; /// The previous speed and direction (used to detect a change) openshot::RendererBase *renderer; int64_t last_video_position; /// The last frame actually displayed int max_sleep_ms; /// The max milliseconds to sleep (when syncing audio and video) diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 524c5b86..ee72d477 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -25,9 +25,9 @@ namespace openshot { // Constructor VideoCacheThread::VideoCacheThread() - : Thread("video-cache"), speed(1), is_playing(false), + : Thread("video-cache"), speed(0), last_speed(1), is_playing(false), reader(NULL), current_display_frame(1), cached_frame_count(0), - min_frames_ahead(12), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6) + min_frames_ahead(24), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6) { } @@ -105,9 +105,15 @@ namespace openshot // Calculate increment (based on speed) // Support caching in both directions - int16_t increment = 1; - if (speed < 0) { - increment = -1; + int16_t increment = speed; + if (speed == 0) { + // When paused, we still want to increment our cache position + // to fully cache frames while paused + if (last_speed > 0) { + increment = 1; + } else { + increment = -1; + } } // Always cache frames from the current display position to our maximum (based on the cache size). @@ -116,11 +122,20 @@ namespace openshot // fragmented cache object (i.e. the user clicking all over the timeline). int64_t starting_frame = current_display_frame; int64_t ending_frame = starting_frame + max_frames_ahead; + + // Adjust ending frame for cache loop if (speed < 0) { + // Reverse loop (if we are going backwards) ending_frame = starting_frame - max_frames_ahead; } + if (ending_frame < 0) { + // Don't allow negative frame number caching + ending_frame = 0; + } // Loop through range of frames (and cache them) + int64_t uncached_frame_count = 0; + int64_t already_cached_frame_count = 0; for (int64_t cache_frame = starting_frame; cache_frame != ending_frame; cache_frame += increment) { cached_frame_count++; if (reader && reader->GetCache() && !reader->GetCache()->Contains(cache_frame)) { @@ -129,9 +144,13 @@ namespace openshot // This frame is not already cached... so request it again (to force the creation & caching) // This will also re-order the missing frame to the front of the cache last_cached_frame = reader->GetFrame(cache_frame); + uncached_frame_count++; } catch (const OutOfBoundsFrame & e) { } + } else if (reader && reader->GetCache() && reader->GetCache()->Contains(cache_frame)) { + already_cached_frame_count++; } + // Check if the user has seeked outside the cache range if (requested_display_frame != current_display_frame) { // cache will restart at a new position @@ -147,11 +166,20 @@ namespace openshot } } - // Update current display frame + // Update cache counts + if (cached_frame_count > max_frames_ahead && uncached_frame_count > (min_frames_ahead / 4)) { + // start cached count again (we have too many uncached frames) + cached_frame_count = 0; + } + + // Update current display frame & last non-paused speed current_display_frame = requested_display_frame; + if (current_speed != 0) { + last_speed = current_speed; + } // Sleep for a fraction of frame duration - std::this_thread::sleep_for(frame_duration / 4); + std::this_thread::sleep_for(frame_duration / 8); } return; diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index dd21bce5..4160b899 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -31,13 +31,14 @@ namespace openshot protected: std::shared_ptr last_cached_frame; int speed; + int last_speed; bool is_playing; int64_t requested_display_frame; int64_t current_display_frame; int64_t cached_frame_count = 0; ReaderBase *reader; - int min_frames_ahead; - int max_frames_ahead; + int64_t min_frames_ahead; + int64_t max_frames_ahead; /// Constructor diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 97d6c221..6b41fbc1 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -758,7 +758,6 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Check cache std::shared_ptr frame; - std::lock_guard guard(get_frame_mutex); frame = final_cache->GetFrame(requested_frame); if (frame) { // Debug output @@ -772,21 +771,7 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Create a scoped lock, allowing only a single thread to run the following code at one time const std::lock_guard lock(getFrameMutex); - // Check for open reader (or throw exception) - if (!is_open) - throw ReaderClosed("The Timeline is closed. Call Open() before calling this method."); - - // Check cache again (due to locking) - frame = final_cache->GetFrame(requested_frame); - if (frame) { - // Debug output - ZmqLogger::Instance()->AppendDebugMethod("Timeline::GetFrame (Cached frame found on 2nd look)", "requested_frame", requested_frame); - - // Return cached frame - return frame; - } - - // Get a list of clips that intersect with the requested section of timeline + // Get a list of clips that intersect with the requested section of timeline // This also opens the readers for intersecting clips, and marks non-intersecting clips as 'needs closing' std::vector nearby_clips; nearby_clips = find_intersecting_clips(requested_frame, 1, true);