diff --git a/src/CacheBase.h b/src/CacheBase.h
index 8ed89935..4377a292 100644
--- a/src/CacheBase.h
+++ b/src/CacheBase.h
@@ -53,6 +53,10 @@ namespace openshot {
/// Clear the cache of all frames
virtual void Clear() = 0;
+ /// @brief Check if frame is already contained in cache
+ /// @param frame_number The frame number to be checked
+ virtual bool Contains(int64_t frame_number) = 0;
+
/// Count the frames in the queue
virtual int64_t Count() = 0;
diff --git a/src/CacheDisk.cpp b/src/CacheDisk.cpp
index bf66bcd9..8cc00003 100644
--- a/src/CacheDisk.cpp
+++ b/src/CacheDisk.cpp
@@ -202,6 +202,15 @@ void CacheDisk::Add(std::shared_ptr frame)
}
}
+// Check if frame is already contained in cache
+bool CacheDisk::Contains(int64_t frame_number) {
+ if (frames.count(frame_number) > 0) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
// Get a frame from the cache (or NULL shared_ptr if no frame is found)
std::shared_ptr CacheDisk::GetFrame(int64_t frame_number)
{
diff --git a/src/CacheDisk.h b/src/CacheDisk.h
index 6c0d1b77..7611728c 100644
--- a/src/CacheDisk.h
+++ b/src/CacheDisk.h
@@ -82,6 +82,10 @@ namespace openshot {
/// Clear the cache of all frames
void Clear();
+ /// @brief Check if frame is already contained in cache
+ /// @param frame_number The frame number to be checked
+ bool Contains(int64_t frame_number);
+
/// Count the frames in the queue
int64_t Count();
diff --git a/src/CacheMemory.cpp b/src/CacheMemory.cpp
index 73c20f89..780d59b1 100644
--- a/src/CacheMemory.cpp
+++ b/src/CacheMemory.cpp
@@ -129,6 +129,15 @@ void CacheMemory::Add(std::shared_ptr frame)
}
}
+// Check if frame is already contained in cache
+bool CacheMemory::Contains(int64_t frame_number) {
+ if (frames.count(frame_number) > 0) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
// Get a frame from the cache (or NULL shared_ptr if no frame is found)
std::shared_ptr CacheMemory::GetFrame(int64_t frame_number)
{
diff --git a/src/CacheMemory.h b/src/CacheMemory.h
index 9be0fdcf..7e5913f0 100644
--- a/src/CacheMemory.h
+++ b/src/CacheMemory.h
@@ -65,6 +65,10 @@ namespace openshot {
/// Clear the cache of all frames
void Clear();
+ /// @brief Check if frame is already contained in cache
+ /// @param frame_number The frame number to be checked
+ bool Contains(int64_t frame_number);
+
/// Count the frames in the queue
int64_t Count();
diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp
index c707f723..ee98ffc0 100644
--- a/src/Qt/PlayerPrivate.cpp
+++ b/src/Qt/PlayerPrivate.cpp
@@ -15,6 +15,7 @@
#include "Exceptions.h"
#include "ZmqLogger.h"
+#include
#include // for std::this_thread::sleep_for
#include // for std::chrono milliseconds, high_resolution_clock
@@ -59,22 +60,38 @@ namespace openshot
// Types for storing time durations in whole and fractional microseconds
using micro_sec = std::chrono::microseconds;
using double_micro_sec = std::chrono::duration;
+ auto sleep_adjustment = double_micro_sec(0.0);
+ std::deque video_diff_history;
+
+ // Init sleep adjustment code
+ float average_video_diff = 0; ///< Average number of frames difference between audio and video thread
+ bool enable_adjustment = false; ///< Toggle desync adjustments ON & OFF (i.e. only adjust when needed)
+ float video_diff_threshold = 1.0; ///< Number of frames difference to trigger an adjustment
while (!threadShouldExit()) {
// Get the start time (to track how long a frame takes to render)
const auto time1 = std::chrono::high_resolution_clock::now();
+
// Calculate on-screen time for a single frame in microseconds
const auto frame_duration = double_micro_sec(1000000.0 / reader->info.fps.ToDouble());
- const auto max_sleep = frame_duration * 1.5;
+ const auto max_sleep = frame_duration * 4; ///< Don't sleep longer than X times a frame duration
+ const auto sleep_adjustment_increment = frame_duration / 100000; ///< Tiny fraction of frame duration
+
+ // Adjust size used in averaging desync calculation
+ auto history_size = reader->info.fps.ToInt(); ///< How many values should we average together (i.e. use FPS)
// Get the current video frame (if it's different)
frame = getFrame();
// Experimental Pausing Code (if frame has not changed)
- if ((speed == 0 && video_position == last_video_position)
- || (video_position > reader->info.video_length)
- ) {
+ if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length))
+ {
speed = 0;
+ // Clear sleep adjustments & average diff
+ sleep_adjustment = double_micro_sec(0.0);
+ average_video_diff = 0.0;
+ video_diff_history.clear();
+ // Sleep for 1 frame
std::this_thread::sleep_for(frame_duration);
continue;
}
@@ -87,15 +104,48 @@ namespace openshot
last_video_position = video_position;
// How many frames ahead or behind is the video thread?
+ // Only calculate this if a reader contains both an audio and video thread
int64_t video_frame_diff = 0;
if (reader->info.has_audio && reader->info.has_video) {
- if (speed != 1)
- // Set audio frame again (since we are not in normal speed, and not paused)
- audioPlayback->Seek(video_position);
-
- // Only calculate this if a reader contains both an audio and video thread
audio_position = audioPlayback->getCurrentFramePosition();
video_frame_diff = video_position - audio_position;
+ // Seek to audio frame again (since we are not in normal speed, and not paused)
+ if (speed != 1) {
+ audioPlayback->Seek(video_position);
+ }
+ }
+
+ // Add video diff to history (for averaging sync delays over time)
+ video_diff_history.push_front(video_frame_diff);
+ if (video_diff_history.size() > history_size) {
+ // Remove oldest element (keep queue frozen with only most recent diffs)
+ video_diff_history.pop_back();
+ // Loop through recent video diff values
+ int64_t diff_sum = 0;
+ for(auto it=video_diff_history.begin(); it!=video_diff_history.end();++it) {
+ diff_sum += *it;
+ }
+ // Calculate average video diff (to be used for syncing video to audio)
+ average_video_diff = float(diff_sum) / history_size;
+ }
+
+ // Should we enable the adjustment? (i.e. too much desync)
+ if (!enable_adjustment && fabs(average_video_diff) > video_diff_threshold) {
+ enable_adjustment = true;
+ } else if (enable_adjustment && fabs(average_video_diff) > 0.0 && fabs(average_video_diff) < 0.25) {
+ // Close to zero, stop adjusting
+ enable_adjustment = false;
+ }
+
+ // Calculate sync adjustment (if adjustment is triggered)
+ if (enable_adjustment) {
+ if (average_video_diff < 0.0) {
+ // We need to slowly decrease the sleep
+ sleep_adjustment -= sleep_adjustment_increment;
+ } else if (video_frame_diff > 0.0) {
+ // We need to slowly increase the sleep
+ sleep_adjustment += sleep_adjustment_increment;
+ }
}
// Get the end time (to track how long a frame takes to render)
@@ -105,13 +155,7 @@ namespace openshot
const auto render_time = double_micro_sec(time2 - time1);
// Calculate the amount of time to sleep (by subtracting the render time)
- auto sleep_time = duration_cast(frame_duration - render_time);
- // Raising video_frame_diff to the power of 3
- // Preserves the sign (+/-) and corrects more aggressively depending on how
- // out of sync the video is.
- sleep_time = sleep_time + std::chrono::microseconds((long(powf(100 * video_frame_diff, 3) )));
-
- ZmqLogger::Instance()->AppendDebugMethod("PlayerPrivate::run (determine sleep)", "video_frame_diff", video_frame_diff, "video_position", video_position, "audio_position", audio_position, "speed", speed, "render_time(ms)", render_time.count(), "sleep_time(ms)", sleep_time.count());
+ auto sleep_time = duration_cast(frame_duration - render_time + sleep_adjustment);
// Sleep if sleep_time is greater than zero after correction
// (leaving the video frame on the screen for the correct amount of time)
@@ -140,7 +184,7 @@ namespace openshot
else
{
// Update cache on which frame was retrieved
- videoCache->setCurrentFramePosition(video_position);
+ videoCache->Seek(video_position);
// return frame from reader
return reader->GetFrame(video_position);
diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp
index 60d94d1d..77f8a78f 100644
--- a/src/Qt/VideoCacheThread.cpp
+++ b/src/Qt/VideoCacheThread.cpp
@@ -26,8 +26,8 @@ namespace openshot
{
// Constructor
VideoCacheThread::VideoCacheThread()
- : Thread("video-cache"), speed(1), is_playing(false), position(1)
- , reader(NULL), max_concurrent_frames(OPEN_MP_NUM_PROCESSORS * 4), current_display_frame(1)
+ : Thread("video-cache"), speed(1), is_playing(false),
+ reader(NULL), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 2), current_display_frame(1)
{
}
@@ -36,25 +36,10 @@ namespace openshot
{
}
- // Get the currently playing frame number (if any)
- int64_t VideoCacheThread::getCurrentFramePosition()
- {
- if (frame)
- return frame->number;
- else
- return 0;
- }
-
- // Set the currently playing frame number (if any)
- void VideoCacheThread::setCurrentFramePosition(int64_t current_frame_number)
- {
- current_display_frame = current_frame_number;
- }
-
// Seek the reader to a particular frame number
void VideoCacheThread::Seek(int64_t new_position)
{
- position = new_position;
+ current_display_frame = new_position;
}
// Play the video
@@ -73,7 +58,6 @@ namespace openshot
void VideoCacheThread::run()
{
// Types for storing time durations in whole and fractional milliseconds
- std::shared_ptr smallest_frame = NULL;
using ms = std::chrono::milliseconds;
using double_ms = std::chrono::duration;
@@ -81,37 +65,52 @@ namespace openshot
// Calculate on-screen time for a single frame in milliseconds
const auto frame_duration = double_ms(1000.0 / reader->info.fps.ToDouble());
- // Cache frames before the other threads need them
- // Cache frames up to the max frames. Reset to current position
- // if cache gets too far away from display frame. Cache frames
- // even when player is paused (i.e. speed 0).
- while (((position - current_display_frame) < max_concurrent_frames) && is_playing)
- {
- // Only cache up till the max_concurrent_frames amount... then sleep
- try
- {
- if (reader) {
- ZmqLogger::Instance()->AppendDebugMethod("VideoCacheThread::run (cache frame)", "position", position, "current_display_frame", current_display_frame, "max_concurrent_frames", max_concurrent_frames, "needed_frames", (position - current_display_frame));
+ // Calculate bytes per frame. If we have a reference openshot::Frame, use that instead (the preview
+ // window can be smaller, can thus reduce the bytes per frame)
+ int64_t bytes_per_frame = (reader->info.height * reader->info.width * 4) +
+ (reader->info.sample_rate * reader->info.channels * 4);
+ if (last_cached_frame && last_cached_frame->has_image_data && last_cached_frame->has_audio_data) {
+ bytes_per_frame = last_cached_frame->GetBytes();
+ }
- // Force the frame to be generated
- smallest_frame = reader->GetCache()->GetSmallestFrame();
- if (smallest_frame && smallest_frame->number > current_display_frame) {
- // Cache position has gotten too far away from current display frame.
- // Reset the position to the current display frame.
- position = current_display_frame;
- }
- reader->GetFrame(position);
- }
+ // Calculate # of frames on Timeline cache
+ if (reader->GetCache() && reader->GetCache()->GetMaxBytes() > 0) {
+ // Use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it)
+ max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2;
+ if (max_frames_ahead > 1000) {
+ // Ignore values that are too large, and default to a safer value
+ max_frames_ahead = OPEN_MP_NUM_PROCESSORS * 2;
+ }
+ }
- }
- catch (const OutOfBoundsFrame & e)
- {
- // Ignore out of bounds frame exceptions
- }
+ // Calculate increment (based on speed)
+ // Support caching in both directions
+ int16_t increment = 1;
+ if (speed < 0) {
+ increment = -1;
+ }
- // Increment frame number
- position++;
- }
+ // Always cache frames from the current display position to our maximum (based on the cache size).
+ // Frames which are already cached are basically free. Only uncached frames have a big CPU cost.
+ // By always looping through the expected frame range, we can fill-in missing frames caused by a
+ // fragmented cache object (i.e. the user clicking all over the timeline).
+ int64_t starting_frame = current_display_frame;
+ int64_t ending_frame = starting_frame + max_frames_ahead;
+ if (speed < 0) {
+ ending_frame = starting_frame - max_frames_ahead;
+ }
+
+ for (int64_t cache_frame = starting_frame; cache_frame != ending_frame; cache_frame += increment) {
+ if (reader && reader->GetCache() && !reader->GetCache()->Contains(cache_frame)) {
+ try
+ {
+ // This frame is not already cached... so request it again (to force the creation & caching)
+ // This will also re-order the missing frame to the front of the cache
+ last_cached_frame = reader->GetFrame(cache_frame);
+ }
+ catch (const OutOfBoundsFrame & e) { }
+ }
+ }
// Sleep for 1 frame length
std::this_thread::sleep_for(frame_duration);
diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h
index 7ef1da8b..cc120b44 100644
--- a/src/Qt/VideoCacheThread.h
+++ b/src/Qt/VideoCacheThread.h
@@ -28,25 +28,19 @@ namespace openshot
*/
class VideoCacheThread : Thread
{
- private:
- std::atomic_int position;
-
protected:
- std::shared_ptr frame;
+ std::shared_ptr last_cached_frame;
int speed;
bool is_playing;
int64_t current_display_frame;
ReaderBase *reader;
- int max_concurrent_frames;
+ int max_frames_ahead;
/// Constructor
VideoCacheThread();
/// Destructor
~VideoCacheThread();
- /// Get the currently playing frame number (if any)
- int64_t getCurrentFramePosition();
-
/// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
int getSpeed() const { return speed; }
@@ -56,9 +50,6 @@ namespace openshot
/// Seek the reader to a particular frame number
void Seek(int64_t new_position);
- /// Set the currently displaying frame number
- void setCurrentFramePosition(int64_t current_frame_number);
-
/// Set Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
void setSpeed(int new_speed) { speed = new_speed; }
diff --git a/src/Timeline.cpp b/src/Timeline.cpp
index 0de6045f..2c95a90c 100644
--- a/src/Timeline.cpp
+++ b/src/Timeline.cpp
@@ -789,15 +789,6 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame)
return frame;
}
- // Check if previous frame was cached? (if not, assume we are seeking somewhere else on the Timeline, and need
- // to clear all cache (for continuity sake). For example, jumping back to a previous spot can cause issues with audio
- // data where the new jump location doesn't match up with the previously cached audio data.
- std::shared_ptr previous_frame = final_cache->GetFrame(requested_frame - 1);
- if (!previous_frame) {
- // Seeking to new place on timeline (destroy cache)
- ClearAllCache();
- }
-
// Minimum number of frames to process (for performance reasons)
int minimum_frames = OPEN_MP_NUM_PROCESSORS;