Adding pre-roll to VideoCacheThread, and making video & audio threads wait for the isReady() method before playing. Also fixed an audio bug where our internal buffer was not being cleared on seek. Removed some unused caching from Clip, and did some minor refactor on FrameMapper/Clip cache clearing.

src/AudioReaderSource.cpp

@@ -20,7 +20,7 @@ using namespace openshot;
 
 // Constructor that reads samples from a reader
 AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number, int buffer_size)
-	: reader(audio_reader), frame_number(starting_frame_number),
+	: reader(audio_reader), frame_number(starting_frame_number), videoCache(NULL),
 	  size(buffer_size), position(0), frame_position(0), estimated_frame(0), speed(1) {
 
 	// Initialize an audio buffer (based on reader)
@@ -168,7 +168,7 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in
 		int number_to_copy = 0;
 
 		// Do we need more samples?
-		if (speed == 1) {
+		if (speed == 1 && videoCache->isReady()) {
 			// Only refill buffers if speed is normal
 			if ((reader && reader->IsOpen() && !frame) or
 				(reader && reader->IsOpen() && buffer_samples - position < info.numSamples))
@@ -177,6 +177,10 @@ void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& in
 		} else {
 			// Fill buffer with silence and clear current frame
 			info.buffer->clear();
+
+			// Empty internal buffer also
+            buffer->clear();
+            position = 0;
 			return;
 		}
 

src/AudioReaderSource.h

@@ -14,6 +14,7 @@
 #define OPENSHOT_AUDIOREADERSOURCE_H
 
 #include "ReaderBase.h"
+#include "Qt/VideoCacheThread.h"
 
 #include <AppConfig.h>
 #include <juce_audio_basics/juce_audio_basics.h>
@@ -42,6 +43,7 @@ namespace openshot
 		int64_t frame_position; /// The position of the current frame's buffer
 		double estimated_frame; /// The estimated frame position of the currently playing buffer
 		int estimated_samples_per_frame; /// The estimated samples per frame of video
+        openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking)
 
 		/// Get more samples from the reader
 		void GetMoreSamplesFromReader();
@@ -103,6 +105,9 @@ namespace openshot
 	    /// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
 	    int getSpeed() const { return speed; }
 
+	    /// Set playback video cache thread (for pre-roll reference)
+	    void setVideoCache(openshot::VideoCacheThread *newCache) { videoCache = newCache; }
+
 	    /// Set Reader
 	    void Reader(ReaderBase *audio_reader) { reader = audio_reader; }
 	    /// Get Reader

src/Clip.cpp

@@ -1198,9 +1198,6 @@ void Clip::AddEffect(EffectBase* effect)
 		}
 	}
     #endif
-
-	// Clear cache
-	cache.Clear();
 }
 
 // Remove an effect from the clip

src/ClipBase.h

@@ -47,8 +47,6 @@ namespace openshot {
 		Json::Value add_property_choice_json(std::string name, int value, int selected_value) const;
 
 	public:
-		CacheMemory cache;
-
 		/// Constructor for the base clip
 		ClipBase() :
 			position(0.0),

src/FrameMapper.cpp

@@ -655,24 +655,24 @@ void FrameMapper::Close()
 
 		// Close internal reader
 		reader->Close();
-
-		// Clear the fields & frames lists
-		fields.clear();
-		frames.clear();
-
-		// Mark as dirty
-		is_dirty = true;
-
-		// Clear cache
-		final_cache.Clear();
-
-		// Deallocate resample buffer
-		if (avr) {
-			SWR_CLOSE(avr);
-			SWR_FREE(&avr);
-			avr = NULL;
-		}
 	}
+
+    // Clear the fields & frames lists
+    fields.clear();
+    frames.clear();
+
+    // Mark as dirty
+    is_dirty = true;
+
+    // Clear cache
+    final_cache.Clear();
+
+    // Deallocate resample buffer
+    if (avr) {
+        SWR_CLOSE(avr);
+        SWR_FREE(&avr);
+        avr = NULL;
+    }
 }
 
 

src/Qt/AudioPlaybackThread.cpp

@@ -92,7 +92,7 @@ namespace openshot
     }
 
     // Constructor
-    AudioPlaybackThread::AudioPlaybackThread()
+    AudioPlaybackThread::AudioPlaybackThread(openshot::VideoCacheThread* cache)
 	: juce::Thread("audio-playback")
 	, player()
 	, transport()
@@ -103,6 +103,7 @@ namespace openshot
     , buffer_size(7000)
     , is_playing(false)
 	, time_thread("audio-buffer")
+	, videoCache(cache)
     {
 	}
 
@@ -125,8 +126,8 @@ namespace openshot
 		sampleRate = reader->info.sample_rate;
 		numChannels = reader->info.channels;
 
-		// TODO: Update transport or audio source's sample rate, incase the sample rate
-		// is different than the original Reader
+		// Set video cache thread
+		source->setVideoCache(videoCache);
 
 		// Mark as 'playing'
 		Play();
@@ -139,12 +140,6 @@ namespace openshot
 	return std::shared_ptr<openshot::Frame>();
     }
 
-    // Get the currently playing frame number
-    int64_t AudioPlaybackThread::getCurrentFramePosition()
-    {
-	return source ? source->getEstimatedFrame() : 0;
-    }
-
 	// Seek the audio thread
 	void AudioPlaybackThread::Seek(int64_t new_position)
 	{

src/Qt/AudioPlaybackThread.h

@@ -19,6 +19,7 @@
 #include "AudioReaderSource.h"
 #include "AudioDevices.h"
 #include "AudioReaderSource.h"
+#include "Qt/VideoCacheThread.h"
 
 #include <OpenShotAudio.h>
 #include <AppConfig.h>
@@ -75,20 +76,22 @@ public:
 		int buffer_size;
 		bool is_playing;
 		juce::TimeSliceThread time_thread;
+        openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking)
+
 		/// Constructor
-		AudioPlaybackThread();
+		AudioPlaybackThread(openshot::VideoCacheThread* cache);
 		/// Destructor
 		~AudioPlaybackThread();
 
 		/// Set the current thread's reader
 		void Reader(openshot::ReaderBase *reader);
 
+		/// Return the current audio transport buffer size (to determine latency)
+		int getBufferSize() { return buffer_size; }
+
 		/// Get the current frame object (which is filling the buffer)
 		std::shared_ptr<openshot::Frame> getFrame();
 
-		/// Get the current frame number being played
-		int64_t getCurrentFramePosition();
-
 		/// Play the audio
 		void Play();
 

src/Qt/PlayerPrivate.cpp

@@ -17,19 +17,20 @@
 
 #include <queue>
 #include <thread>    // for std::this_thread::sleep_for
-#include <chrono>    // for std::chrono milliseconds, high_resolution_clock
+#include <chrono>    // for std::chrono microseconds, high_resolution_clock
 
 namespace openshot
 {
     int close_to_sync = 5;
     // Constructor
     PlayerPrivate::PlayerPrivate(openshot::RendererBase *rb)
-    : renderer(rb), Thread("player"), video_position(1), audio_position(0)
-    , audioPlayback(new openshot::AudioPlaybackThread())
-    , videoPlayback(new openshot::VideoPlaybackThread(rb))
-    , videoCache(new openshot::VideoCacheThread())
-    , speed(1), reader(NULL), last_video_position(1), max_sleep_ms(125000)
-    { }
+    : renderer(rb), Thread("player"), video_position(1), audio_position(0),
+      speed(1), reader(NULL), last_video_position(1), max_sleep_ms(125000), playback_frames(0)
+    {
+        videoCache = new openshot::VideoCacheThread();
+        audioPlayback = new openshot::AudioPlaybackThread(videoCache);
+        videoPlayback = new openshot::VideoPlaybackThread(rb);
+    }
 
     // Destructor
     PlayerPrivate::~PlayerPrivate()
@@ -61,26 +62,34 @@ namespace openshot
         using micro_sec = std::chrono::microseconds;
         using double_micro_sec = std::chrono::duration<double, micro_sec::period>;
 
-        // Time-based video sync
-        auto start_time = std::chrono::high_resolution_clock::now(); ///< timestamp playback starts
+        // Calculate latency of audio thread (i.e. how many microseconds before samples are audible)
+        // TODO: This is the experimental amount of latency I have on my system audio playback
+        //const auto audio_latency = double_micro_sec(1000000.0 * (audioPlayback->getBufferSize() / reader->info.sample_rate));
+        const auto audio_latency = double_micro_sec(240000);
+
+        // Init start_time of playback
+        std::chrono::time_point<std::chrono::system_clock, std::chrono::microseconds> start_time;
+        start_time = std::chrono::time_point_cast<micro_sec>(std::chrono::high_resolution_clock::now()); ///< timestamp playback starts
 
         while (!threadShouldExit()) {
             // Calculate on-screen time for a single frame
             const auto frame_duration = double_micro_sec(1000000.0 / reader->info.fps.ToDouble());
             const auto max_sleep = frame_duration * 4; ///< Don't sleep longer than X times a frame duration
 
-            // Get the current video frame (if it's different)
+            // Get the current video frame
             frame = getFrame();
 
             // Pausing Code (if frame has not changed)
-            if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length))
+            if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length) || !videoCache->isReady())
             {
-                // Set start time to prepare for next playback period and reset frame counter
-                start_time = std::chrono::high_resolution_clock::now();
-                playback_frames = 0;
-
                 // Sleep for a fraction of frame duration
                 std::this_thread::sleep_for(frame_duration / 4);
+                audioPlayback->Seek(video_position);
+
+                // Reset current playback start time
+                start_time = std::chrono::time_point_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now());
+                playback_frames = 0;
+
                 continue;
             }
 
@@ -91,21 +100,10 @@ namespace openshot
             // Keep track of the last displayed frame
             last_video_position = video_position;
 
-            // How many frames ahead or behind is the video thread?
-            // Only calculate this if a reader contains both an audio and video thread
-            int64_t video_frame_diff = 0;
-            if (reader->info.has_audio && reader->info.has_video) {
-                audio_position = audioPlayback->getCurrentFramePosition();
-                video_frame_diff = video_position - audio_position;
-                // Seek to audio frame again (since we are not in normal speed, and not paused)
-                if (speed != 1) {
-                    audioPlayback->Seek(video_position);
-                }
-            }
-
             // Calculate the diff between 'now' and the predicted frame end time
             const auto current_time = std::chrono::high_resolution_clock::now();
-            const auto remaining_time = double_micro_sec(start_time + (frame_duration * playback_frames) - current_time);
+            const auto remaining_time = double_micro_sec(start_time + audio_latency +
+                    (frame_duration * playback_frames) - current_time);
 
             // Sleep to display video image on screen
             if (remaining_time > remaining_time.zero() ) {
@@ -151,6 +149,13 @@ namespace openshot
     return std::shared_ptr<openshot::Frame>();
     }
 
+    // Seek to a new position
+    void PlayerPrivate::Seek(int64_t new_position)
+    {
+        video_position = new_position;
+        last_video_position = 0;
+    }
+
     // Start video/audio playback
     bool PlayerPrivate::startPlayback()
     {

src/Qt/PlayerPrivate.h

@@ -30,7 +30,7 @@ namespace openshot
     class PlayerPrivate : juce::Thread
     {
     std::shared_ptr<openshot::Frame> frame; /// The current frame
-    int64_t playback_frames = 0; /// The # of frames since playback started
+    int64_t playback_frames; /// The # of frames since playback started
 	int64_t video_position; /// The current frame position.
 	int64_t audio_position; /// The current frame position.
 	openshot::ReaderBase *reader; /// The reader which powers this player
@@ -53,6 +53,9 @@ namespace openshot
 	/// Start the video/audio playback
 	bool startPlayback();
 
+	/// Seek to a new frame #
+	void Seek(int64_t new_position);
+
 	/// Stop the video/audio playback
 	void stopPlayback();
 

src/Qt/VideoCacheThread.cpp

@@ -27,7 +27,8 @@ namespace openshot
 	// Constructor
 	VideoCacheThread::VideoCacheThread()
 	: Thread("video-cache"), speed(1), is_playing(false),
-	reader(NULL), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 2), current_display_frame(1)
+	reader(NULL), current_display_frame(1), cached_frame_count(0),
+	min_frames_ahead(12), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6)
     {
     }
 
@@ -39,9 +40,18 @@ namespace openshot
 	// Seek the reader to a particular frame number
 	void VideoCacheThread::Seek(int64_t new_position)
 	{
-		current_display_frame = new_position;
+        requested_display_frame = new_position;
 	}
 
+    // Seek the reader to a particular frame number and optionally start the pre-roll
+    void VideoCacheThread::Seek(int64_t new_position, bool start_preroll)
+    {
+	    if (start_preroll && reader && reader->GetCache() && !reader->GetCache()->Contains(new_position)) {
+            cached_frame_count = 0;
+	    }
+        Seek(new_position);
+    }
+
 	// Play the video
 	void VideoCacheThread::Play() {
 		// Start playing
@@ -54,6 +64,11 @@ namespace openshot
 		is_playing = false;
 	}
 
+	// Is cache ready for playback (pre-roll)
+    bool VideoCacheThread::isReady() {
+	    return (cached_frame_count > min_frames_ahead);
+	}
+
     // Start the thread
     void VideoCacheThread::run()
     {
@@ -73,13 +88,18 @@ namespace openshot
                 bytes_per_frame = last_cached_frame->GetBytes();
             }
 
-            // Calculate # of frames on Timeline cache
+            // Calculate # of frames on Timeline cache (when paused)
             if (reader->GetCache() && reader->GetCache()->GetMaxBytes() > 0) {
-                // Use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it)
-                max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2;
-                if (max_frames_ahead > 1000) {
-                    // Ignore values that are too large, and default to a safer value
-                    max_frames_ahead = OPEN_MP_NUM_PROCESSORS * 2;
+                if (speed == 0) {
+                    // When paused, use 1/2 the cache size (so our cache will be 50% before the play-head, and 50% after it)
+                    max_frames_ahead = (reader->GetCache()->GetMaxBytes() / bytes_per_frame) / 2;
+                    if (max_frames_ahead > 300) {
+                        // Ignore values that are too large, and default to a safer value
+                        max_frames_ahead = 300;
+                    }
+                } else {
+                    // When playing back video (speed == 1), keep cache # small
+                    max_frames_ahead = min_frames_ahead;
                 }
             }
 
@@ -100,7 +120,9 @@ namespace openshot
                 ending_frame = starting_frame - max_frames_ahead;
             }
 
+            // Loop through range of frames (and cache them)
             for (int64_t cache_frame = starting_frame; cache_frame != ending_frame; cache_frame += increment) {
+                cached_frame_count++;
                 if (reader && reader->GetCache() && !reader->GetCache()->Contains(cache_frame)) {
                     try
                     {
@@ -110,8 +132,20 @@ namespace openshot
                     }
                     catch (const OutOfBoundsFrame & e) {  }
                 }
+                // Check if the user has seeked outside the cache range
+                if (requested_display_frame != current_display_frame) {
+                    // cache will restart at a new position
+                    if (speed >= 0 && (requested_display_frame < starting_frame || requested_display_frame > ending_frame)) {
+                        break;
+                    } else if (speed < 0 && (requested_display_frame > starting_frame || requested_display_frame < ending_frame)) {
+                        break;
+                    }
+                }
             }
 
+            // Update current display frame
+            current_display_frame = requested_display_frame;
+
 			// Sleep for a fraction of frame duration
 			std::this_thread::sleep_for(frame_duration / 4);
 		}

src/Qt/VideoCacheThread.h

@@ -32,10 +32,14 @@ namespace openshot
 	std::shared_ptr<Frame> last_cached_frame;
 	int speed;
 	bool is_playing;
+	int64_t requested_display_frame;
 	int64_t current_display_frame;
+	int64_t cached_frame_count = 0;
 	ReaderBase *reader;
+	int min_frames_ahead;
 	int max_frames_ahead;
 
+
 	/// Constructor
 	VideoCacheThread();
 	/// Destructor
@@ -47,8 +51,11 @@ namespace openshot
 	/// Play the video
 	void Play();
 
-	/// Seek the reader to a particular frame number
-	void Seek(int64_t new_position);
+    /// Seek the reader to a particular frame number
+    void Seek(int64_t new_position);
+
+	/// Seek the reader to a particular frame number and optionally start the pre-roll
+	void Seek(int64_t new_position, bool start_preroll);
 
     /// Set Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
     void setSpeed(int new_speed) { speed = new_speed; }
@@ -65,8 +72,11 @@ namespace openshot
 	/// Parent class of VideoCacheThread
 	friend class PlayerPrivate;
 	friend class QtPlayer;
-    };
 
+    public:
+        /// Is cache ready for video/audio playback
+        bool isReady();
+    };
 }
 
 #endif // OPENSHOT_VIDEO_CACHE_THREAD_H

src/QtPlayer.cpp

@@ -155,7 +155,7 @@ namespace openshot
     	// Check for seek
     	if (reader && threads_started && new_frame > 0) {
     		// Notify cache thread that seek has occurred
-    		p->videoCache->Seek(new_frame);
+    		p->videoCache->Seek(new_frame, true);
 
             // Notify audio thread that seek has occurred
             p->audioPlayback->Seek(new_frame);

src/Timeline.cpp

@@ -343,7 +343,6 @@ void Timeline::AddClip(Clip* clip)
 	clip->ParentTimeline(this);
 
 	// Clear cache of clip and nested reader (if any)
-	clip->cache.Clear();
 	if (clip->Reader() && clip->Reader()->GetCache())
 		clip->Reader()->GetCache()->Clear();
 
@@ -734,9 +733,8 @@ void Timeline::Close()
 	// Mark timeline as closed
 	is_open = false;
 
-	// Clear cache
-	if (final_cache)
-		final_cache->Clear();
+	// Clear all cache
+    ClearAllCache();
 }
 
 // Open the reader (and start consuming resources)
@@ -754,7 +752,6 @@ bool Timeline::isEqual(double a, double b)
 // Get an openshot::Frame object for a specific frame number of this reader.
 std::shared_ptr<Frame> Timeline::GetFrame(int64_t requested_frame)
 {
-
 	// Adjust out of bounds frame number
 	if (requested_frame < 1)
 		requested_frame = 1;
@@ -789,9 +786,6 @@ std::shared_ptr<Frame> Timeline::GetFrame(int64_t requested_frame)
 			return frame;
 		}
 
-		// Minimum number of frames to process (for performance reasons)
-		int minimum_frames = OPEN_MP_NUM_PROCESSORS;
-
 		// Get a list of clips that intersect with the requested section of timeline
 		// This also opens the readers for intersecting clips, and marks non-intersecting clips as 'needs closing'
 		std::vector<Clip*> nearby_clips;
@@ -1474,7 +1468,9 @@ void Timeline::ClearAllCache() {
     const std::lock_guard<std::recursive_mutex> lock(getFrameMutex);
 
     // Clear primary cache
-    final_cache->Clear();
+    if (final_cache) {
+        final_cache->Clear();
+    }
 
     // Loop through all clips
     for (auto clip : clips)