diff --git a/src/AudioReaderSource.cpp b/src/AudioReaderSource.cpp index 67359a69..a9fb7aa2 100644 --- a/src/AudioReaderSource.cpp +++ b/src/AudioReaderSource.cpp @@ -20,7 +20,7 @@ using namespace openshot; // Constructor that reads samples from a reader AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number, int buffer_size) - : reader(audio_reader), frame_number(starting_frame_number), + : reader(audio_reader), frame_number(starting_frame_number), videoCache(NULL), size(buffer_size), position(0), frame_position(0), estimated_frame(0), speed(1) { // Initialize an audio buffer (based on reader) @@ -168,7 +168,7 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in int number_to_copy = 0; // Do we need more samples? - if (speed == 1) { + if (speed == 1 && videoCache->isReady()) { // Only refill buffers if speed is normal if ((reader && reader->IsOpen() && !frame) or (reader && reader->IsOpen() && buffer_samples - position < info.numSamples)) @@ -177,6 +177,10 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in } else { // Fill buffer with silence and clear current frame info.buffer->clear(); + + // Empty internal buffer also + buffer->clear(); + position = 0; return; } diff --git a/src/AudioReaderSource.h b/src/AudioReaderSource.h index 32356567..81301f4e 100644 --- a/src/AudioReaderSource.h +++ b/src/AudioReaderSource.h @@ -14,6 +14,7 @@ #define OPENSHOT_AUDIOREADERSOURCE_H #include "ReaderBase.h" +#include "Qt/VideoCacheThread.h" #include #include @@ -42,6 +43,7 @@ namespace openshot int64_t frame_position; /// The position of the current frame's buffer double estimated_frame; /// The estimated frame position of the currently playing buffer int estimated_samples_per_frame; /// The estimated samples per frame of video + openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking) /// Get more samples from the reader void GetMoreSamplesFromReader(); @@ -103,6 +105,9 @@ namespace openshot /// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) int getSpeed() const { return speed; } + /// Set playback video cache thread (for pre-roll reference) + void setVideoCache(openshot::VideoCacheThread *newCache) { videoCache = newCache; } + /// Set Reader void Reader(ReaderBase *audio_reader) { reader = audio_reader; } /// Get Reader diff --git a/src/Clip.cpp b/src/Clip.cpp index 7fdd9555..b323f657 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -1198,9 +1198,6 @@ void Clip::AddEffect(EffectBase* effect) } } #endif - - // Clear cache - cache.Clear(); } // Remove an effect from the clip diff --git a/src/ClipBase.h b/src/ClipBase.h index 83054311..9ac5cb44 100644 --- a/src/ClipBase.h +++ b/src/ClipBase.h @@ -47,8 +47,6 @@ namespace openshot { Json::Value add_property_choice_json(std::string name, int value, int selected_value) const; public: - CacheMemory cache; - /// Constructor for the base clip ClipBase() : position(0.0), diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index c405bf26..74666755 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -655,24 +655,24 @@ void FrameMapper::Close() // Close internal reader reader->Close(); - - // Clear the fields & frames lists - fields.clear(); - frames.clear(); - - // Mark as dirty - is_dirty = true; - - // Clear cache - final_cache.Clear(); - - // Deallocate resample buffer - if (avr) { - SWR_CLOSE(avr); - SWR_FREE(&avr); - avr = NULL; - } } + + // Clear the fields & frames lists + fields.clear(); + frames.clear(); + + // Mark as dirty + is_dirty = true; + + // Clear cache + final_cache.Clear(); + + // Deallocate resample buffer + if (avr) { + SWR_CLOSE(avr); + SWR_FREE(&avr); + avr = NULL; + } } diff --git a/src/Qt/AudioPlaybackThread.cpp b/src/Qt/AudioPlaybackThread.cpp index b13ce5a3..d1654458 100644 --- a/src/Qt/AudioPlaybackThread.cpp +++ b/src/Qt/AudioPlaybackThread.cpp @@ -92,7 +92,7 @@ namespace openshot } // Constructor - AudioPlaybackThread::AudioPlaybackThread() + AudioPlaybackThread::AudioPlaybackThread(openshot::VideoCacheThread* cache) : juce::Thread("audio-playback") , player() , transport() @@ -103,6 +103,7 @@ namespace openshot , buffer_size(7000) , is_playing(false) , time_thread("audio-buffer") + , videoCache(cache) { } @@ -125,8 +126,8 @@ namespace openshot sampleRate = reader->info.sample_rate; numChannels = reader->info.channels; - // TODO: Update transport or audio source's sample rate, incase the sample rate - // is different than the original Reader + // Set video cache thread + source->setVideoCache(videoCache); // Mark as 'playing' Play(); @@ -139,12 +140,6 @@ namespace openshot return std::shared_ptr(); } - // Get the currently playing frame number - int64_t AudioPlaybackThread::getCurrentFramePosition() - { - return source ? source->getEstimatedFrame() : 0; - } - // Seek the audio thread void AudioPlaybackThread::Seek(int64_t new_position) { diff --git a/src/Qt/AudioPlaybackThread.h b/src/Qt/AudioPlaybackThread.h index ac9819da..821b7138 100644 --- a/src/Qt/AudioPlaybackThread.h +++ b/src/Qt/AudioPlaybackThread.h @@ -19,6 +19,7 @@ #include "AudioReaderSource.h" #include "AudioDevices.h" #include "AudioReaderSource.h" +#include "Qt/VideoCacheThread.h" #include #include @@ -75,20 +76,22 @@ public: int buffer_size; bool is_playing; juce::TimeSliceThread time_thread; + openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking) + /// Constructor - AudioPlaybackThread(); + AudioPlaybackThread(openshot::VideoCacheThread* cache); /// Destructor ~AudioPlaybackThread(); /// Set the current thread's reader void Reader(openshot::ReaderBase *reader); + /// Return the current audio transport buffer size (to determine latency) + int getBufferSize() { return buffer_size; } + /// Get the current frame object (which is filling the buffer) std::shared_ptr getFrame(); - /// Get the current frame number being played - int64_t getCurrentFramePosition(); - /// Play the audio void Play(); diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 969200b8..cf63c8e0 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -17,19 +17,20 @@ #include #include // for std::this_thread::sleep_for -#include // for std::chrono milliseconds, high_resolution_clock +#include // for std::chrono microseconds, high_resolution_clock namespace openshot { int close_to_sync = 5; // Constructor PlayerPrivate::PlayerPrivate(openshot::RendererBase *rb) - : renderer(rb), Thread("player"), video_position(1), audio_position(0) - , audioPlayback(new openshot::AudioPlaybackThread()) - , videoPlayback(new openshot::VideoPlaybackThread(rb)) - , videoCache(new openshot::VideoCacheThread()) - , speed(1), reader(NULL), last_video_position(1), max_sleep_ms(125000) - { } + : renderer(rb), Thread("player"), video_position(1), audio_position(0), + speed(1), reader(NULL), last_video_position(1), max_sleep_ms(125000), playback_frames(0) + { + videoCache = new openshot::VideoCacheThread(); + audioPlayback = new openshot::AudioPlaybackThread(videoCache); + videoPlayback = new openshot::VideoPlaybackThread(rb); + } // Destructor PlayerPrivate::~PlayerPrivate() @@ -61,26 +62,34 @@ namespace openshot using micro_sec = std::chrono::microseconds; using double_micro_sec = std::chrono::duration; - // Time-based video sync - auto start_time = std::chrono::high_resolution_clock::now(); ///< timestamp playback starts + // Calculate latency of audio thread (i.e. how many microseconds before samples are audible) + // TODO: This is the experimental amount of latency I have on my system audio playback + //const auto audio_latency = double_micro_sec(1000000.0 * (audioPlayback->getBufferSize() / reader->info.sample_rate)); + const auto audio_latency = double_micro_sec(240000); + + // Init start_time of playback + std::chrono::time_point start_time; + start_time = std::chrono::time_point_cast(std::chrono::high_resolution_clock::now()); ///< timestamp playback starts while (!threadShouldExit()) { // Calculate on-screen time for a single frame const auto frame_duration = double_micro_sec(1000000.0 / reader->info.fps.ToDouble()); const auto max_sleep = frame_duration * 4; ///< Don't sleep longer than X times a frame duration - // Get the current video frame (if it's different) + // Get the current video frame frame = getFrame(); // Pausing Code (if frame has not changed) - if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length)) + if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length) || !videoCache->isReady()) { - // Set start time to prepare for next playback period and reset frame counter - start_time = std::chrono::high_resolution_clock::now(); - playback_frames = 0; - // Sleep for a fraction of frame duration std::this_thread::sleep_for(frame_duration / 4); + audioPlayback->Seek(video_position); + + // Reset current playback start time + start_time = std::chrono::time_point_cast(std::chrono::high_resolution_clock::now()); + playback_frames = 0; + continue; } @@ -91,21 +100,10 @@ namespace openshot // Keep track of the last displayed frame last_video_position = video_position; - // How many frames ahead or behind is the video thread? - // Only calculate this if a reader contains both an audio and video thread - int64_t video_frame_diff = 0; - if (reader->info.has_audio && reader->info.has_video) { - audio_position = audioPlayback->getCurrentFramePosition(); - video_frame_diff = video_position - audio_position; - // Seek to audio frame again (since we are not in normal speed, and not paused) - if (speed != 1) { - audioPlayback->Seek(video_position); - } - } - // Calculate the diff between 'now' and the predicted frame end time const auto current_time = std::chrono::high_resolution_clock::now(); - const auto remaining_time = double_micro_sec(start_time + (frame_duration * playback_frames) - current_time); + const auto remaining_time = double_micro_sec(start_time + audio_latency + + (frame_duration * playback_frames) - current_time); // Sleep to display video image on screen if (remaining_time > remaining_time.zero() ) { @@ -151,6 +149,13 @@ namespace openshot return std::shared_ptr(); } + // Seek to a new position + void PlayerPrivate::Seek(int64_t new_position) + { + video_position = new_position; + last_video_position = 0; + } + // Start video/audio playback bool PlayerPrivate::startPlayback() { diff --git a/src/Qt/PlayerPrivate.h b/src/Qt/PlayerPrivate.h index a2a8644c..107341d6 100644 --- a/src/Qt/PlayerPrivate.h +++ b/src/Qt/PlayerPrivate.h @@ -30,7 +30,7 @@ namespace openshot class PlayerPrivate : juce::Thread { std::shared_ptr frame; /// The current frame - int64_t playback_frames = 0; /// The # of frames since playback started + int64_t playback_frames; /// The # of frames since playback started int64_t video_position; /// The current frame position. int64_t audio_position; /// The current frame position. openshot::ReaderBase *reader; /// The reader which powers this player @@ -53,6 +53,9 @@ namespace openshot /// Start the video/audio playback bool startPlayback(); + /// Seek to a new frame # + void Seek(int64_t new_position); + /// Stop the video/audio playback void stopPlayback(); diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 4edf7a18..f1ddf42a 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -27,7 +27,8 @@ namespace openshot // Constructor VideoCacheThread::VideoCacheThread() : Thread("video-cache"), speed(1), is_playing(false), - reader(NULL), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 2), current_display_frame(1) + reader(NULL), current_display_frame(1), cached_frame_count(0), + min_frames_ahead(12), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6) { } @@ -39,9 +40,18 @@ namespace openshot // Seek the reader to a particular frame number void VideoCacheThread::Seek(int64_t new_position) { - current_display_frame = new_position; + requested_display_frame = new_position; } + // Seek the reader to a particular frame number and optionally start the pre-roll + void VideoCacheThread::Seek(int64_t new_position, bool start_preroll) + { + if (start_preroll && reader && reader->GetCache() && !reader->GetCache()->Contains(new_position)) { + cached_frame_count = 0; + } + Seek(new_position); + } + // Play the video void VideoCacheThread::Play() { // Start playing @@ -54,6 +64,11 @@ namespace openshot is_playing = false; } + // Is cache ready for playback (pre-roll) + bool VideoCacheThread::isReady() { + return (cached_frame_count > min_frames_ahead); + } + // Start the thread void VideoCacheThread::run() { @@ -73,13 +88,18 @@ namespace openshot bytes_per_frame = last_cached_frame->GetBytes(); } - // Calculate # of frames on Timeline cache + // Calculate # of frames on Timeline cache (when paused) if (reader->GetCache() && reader->GetCache()->GetMaxBytes() > 0) { - // Use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it) - max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2; - if (max_frames_ahead > 1000) { - // Ignore values that are too large, and default to a safer value - max_frames_ahead = OPEN_MP_NUM_PROCESSORS * 2; + if (speed == 0) { + // When paused, use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it) + max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2; + if (max_frames_ahead > 300) { + // Ignore values that are too large, and default to a safer value + max_frames_ahead = 300; + } + } else { + // When playing back video (speed == 1), keep cache # small + max_frames_ahead = min_frames_ahead; } } @@ -100,7 +120,9 @@ namespace openshot ending_frame = starting_frame - max_frames_ahead; } + // Loop through range of frames (and cache them) for (int64_t cache_frame = starting_frame; cache_frame != ending_frame; cache_frame += increment) { + cached_frame_count++; if (reader && reader->GetCache() && !reader->GetCache()->Contains(cache_frame)) { try { @@ -110,8 +132,20 @@ namespace openshot } catch (const OutOfBoundsFrame & e) { } } + // Check if the user has seeked outside the cache range + if (requested_display_frame != current_display_frame) { + // cache will restart at a new position + if (speed >= 0 && (requested_display_frame < starting_frame || requested_display_frame > ending_frame)) { + break; + } else if (speed < 0 && (requested_display_frame > starting_frame || requested_display_frame < ending_frame)) { + break; + } + } } + // Update current display frame + current_display_frame = requested_display_frame; + // Sleep for a fraction of frame duration std::this_thread::sleep_for(frame_duration / 4); } diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index cc120b44..dd21bce5 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -32,10 +32,14 @@ namespace openshot std::shared_ptr last_cached_frame; int speed; bool is_playing; + int64_t requested_display_frame; int64_t current_display_frame; + int64_t cached_frame_count = 0; ReaderBase *reader; + int min_frames_ahead; int max_frames_ahead; + /// Constructor VideoCacheThread(); /// Destructor @@ -47,8 +51,11 @@ namespace openshot /// Play the video void Play(); - /// Seek the reader to a particular frame number - void Seek(int64_t new_position); + /// Seek the reader to a particular frame number + void Seek(int64_t new_position); + + /// Seek the reader to a particular frame number and optionally start the pre-roll + void Seek(int64_t new_position, bool start_preroll); /// Set Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) void setSpeed(int new_speed) { speed = new_speed; } @@ -65,8 +72,11 @@ namespace openshot /// Parent class of VideoCacheThread friend class PlayerPrivate; friend class QtPlayer; - }; + public: + /// Is cache ready for video/audio playback + bool isReady(); + }; } #endif // OPENSHOT_VIDEO_CACHE_THREAD_H diff --git a/src/QtPlayer.cpp b/src/QtPlayer.cpp index 13e07c77..9f148572 100644 --- a/src/QtPlayer.cpp +++ b/src/QtPlayer.cpp @@ -155,7 +155,7 @@ namespace openshot // Check for seek if (reader && threads_started && new_frame > 0) { // Notify cache thread that seek has occurred - p->videoCache->Seek(new_frame); + p->videoCache->Seek(new_frame, true); // Notify audio thread that seek has occurred p->audioPlayback->Seek(new_frame); diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 2c95a90c..97d6c221 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -343,7 +343,6 @@ void Timeline::AddClip(Clip* clip) clip->ParentTimeline(this); // Clear cache of clip and nested reader (if any) - clip->cache.Clear(); if (clip->Reader() && clip->Reader()->GetCache()) clip->Reader()->GetCache()->Clear(); @@ -734,9 +733,8 @@ void Timeline::Close() // Mark timeline as closed is_open = false; - // Clear cache - if (final_cache) - final_cache->Clear(); + // Clear all cache + ClearAllCache(); } // Open the reader (and start consuming resources) @@ -754,7 +752,6 @@ bool Timeline::isEqual(double a, double b) // Get an openshot::Frame object for a specific frame number of this reader. std::shared_ptr Timeline::GetFrame(int64_t requested_frame) { - // Adjust out of bounds frame number if (requested_frame < 1) requested_frame = 1; @@ -789,9 +786,6 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) return frame; } - // Minimum number of frames to process (for performance reasons) - int minimum_frames = OPEN_MP_NUM_PROCESSORS; - // Get a list of clips that intersect with the requested section of timeline // This also opens the readers for intersecting clips, and marks non-intersecting clips as 'needs closing' std::vector nearby_clips; @@ -1474,7 +1468,9 @@ void Timeline::ClearAllCache() { const std::lock_guard lock(getFrameMutex); // Clear primary cache - final_cache->Clear(); + if (final_cache) { + final_cache->Clear(); + } // Loop through all clips for (auto clip : clips)