diff --git a/src/AudioReaderSource.cpp b/src/AudioReaderSource.cpp index a9fb7aa2..c4d34dde 100644 --- a/src/AudioReaderSource.cpp +++ b/src/AudioReaderSource.cpp @@ -13,217 +13,82 @@ #include "AudioReaderSource.h" #include "Exceptions.h" #include "Frame.h" -#include "ZmqLogger.h" + using namespace std; using namespace openshot; // Constructor that reads samples from a reader -AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number, int buffer_size) - : reader(audio_reader), frame_number(starting_frame_number), videoCache(NULL), - size(buffer_size), position(0), frame_position(0), estimated_frame(0), speed(1) { - - // Initialize an audio buffer (based on reader) - buffer = new juce::AudioBuffer(reader->info.channels, size); - - // initialize the audio samples to zero (silence) - buffer->clear(); +AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number) + : reader(audio_reader), frame_position(starting_frame_number), videoCache(NULL), frame(NULL), + sample_position(0), speed(1), stream_position(0) { } // Destructor AudioReaderSource::~AudioReaderSource() { - delete buffer; - buffer = NULL; -} - -// Get more samples from the reader -void AudioReaderSource::GetMoreSamplesFromReader() -{ - // Determine the amount of samples needed to fill up this buffer - int amount_needed = position; // replace these used samples - int amount_remaining = size - amount_needed; // these are unused samples, and need to be carried forward - if (!frame) { - // If no frame, load entire buffer - amount_needed = size; - amount_remaining = 0; - } - - // Debug - ZmqLogger::Instance()->AppendDebugMethod("AudioReaderSource::GetMoreSamplesFromReader", "amount_needed", amount_needed, "amount_remaining", amount_remaining); - - // Init estimated buffer equal to the current frame position (before getting more samples) - estimated_frame = frame_number; - - // Init new buffer - auto *new_buffer = new juce::AudioBuffer(reader->info.channels, size); - new_buffer->clear(); - - // Move the remaining samples into new buffer (if any) - if (amount_remaining > 0) { - for (int channel = 0; channel < buffer->getNumChannels(); channel++) - new_buffer->addFrom(channel, 0, *buffer, channel, position, amount_remaining); - - position = amount_remaining; - } else - // reset position to 0 - position = 0; - - // Loop through frames until buffer filled - while (amount_needed > 0 && speed == 1 && frame_number >= 1 && frame_number <= reader->info.video_length) { - - // Get the next frame (if position is zero) - if (frame_position == 0) { - try { - // Get frame object - frame = reader->GetFrame(frame_number); - frame_number = frame_number + speed; - - } catch (const ReaderClosed & e) { - break; - } catch (const OutOfBoundsFrame & e) { - break; - } - } - - bool frame_completed = false; - int amount_to_copy = 0; - if (frame) - amount_to_copy = frame->GetAudioSamplesCount() - frame_position; - if (amount_to_copy > amount_needed) { - // Don't copy too many samples (we don't want to overflow the buffer) - amount_to_copy = amount_needed; - amount_needed = 0; - } else { - // Not enough to fill the buffer (so use the entire frame) - amount_needed -= amount_to_copy; - frame_completed = true; - } - - // Load all of its samples into the buffer - if (frame) - for (int channel = 0; channel < new_buffer->getNumChannels(); channel++) - new_buffer->addFrom(channel, position, *frame->GetAudioSampleBuffer(), channel, frame_position, amount_to_copy); - - // Adjust remaining samples - position += amount_to_copy; - if (frame_completed) - // Reset frame buffer position (which will load a new frame on the next loop) - frame_position = 0; - else - // Continue tracking the current frame's position - frame_position += amount_to_copy; - } - - // Delete old buffer - buffer->clear(); - delete buffer; - - // Replace buffer and reset position - buffer = new_buffer; - position = 0; -} - -// Reverse an audio buffer -juce::AudioBuffer* AudioReaderSource::reverse_buffer(juce::AudioBuffer* buffer) -{ - int number_of_samples = buffer->getNumSamples(); - int channels = buffer->getNumChannels(); - - // Debug - ZmqLogger::Instance()->AppendDebugMethod("AudioReaderSource::reverse_buffer", "number_of_samples", number_of_samples, "channels", channels); - - // Reverse array (create new buffer to hold the reversed version) - auto *reversed = new juce::AudioBuffer(channels, number_of_samples); - reversed->clear(); - - for (int channel = 0; channel < channels; channel++) - { - int n=0; - for (int s = number_of_samples - 1; s >= 0; s--, n++) - reversed->getWritePointer(channel)[n] = buffer->getWritePointer(channel)[s]; - } - - // Copy the samples back to the original array - buffer->clear(); - // Loop through channels, and get audio samples - for (int channel = 0; channel < channels; channel++) - // Get the audio samples for this channel - buffer->addFrom(channel, 0, reversed->getReadPointer(channel), number_of_samples, 1.0f); - - delete reversed; - reversed = NULL; - - // return pointer or passed in object (so this method can be chained together) - return buffer; } // Get the next block of audio samples void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& info) { - int buffer_samples = buffer->getNumSamples(); - int buffer_channels = buffer->getNumChannels(); - if (info.numSamples > 0) { - int number_to_copy = 0; + int remaining_samples = info.numSamples; + int remaining_position = info.startSample; - // Do we need more samples? - if (speed == 1 && videoCache->isReady()) { - // Only refill buffers if speed is normal - if ((reader && reader->IsOpen() && !frame) or - (reader && reader->IsOpen() && buffer_samples - position < info.numSamples)) - // Refill buffer from reader - GetMoreSamplesFromReader(); - } else { - // Fill buffer with silence and clear current frame + // Pause and fill buffer with silence (wait for pre-roll) + if (speed != 1 || !videoCache->isReady()) { info.buffer->clear(); - - // Empty internal buffer also - buffer->clear(); - position = 0; return; } - // Determine how many samples to copy - if (position + info.numSamples <= buffer_samples) - { - // copy the full amount requested - number_to_copy = info.numSamples; - } - else if (position > buffer_samples) - { - // copy nothing - number_to_copy = 0; - } - else if (buffer_samples - position > 0) - { - // only copy what is left in the buffer - number_to_copy = buffer_samples - position; - } - else - { - // copy nothing - number_to_copy = 0; - } + while (remaining_samples > 0) { + try { + // Get current frame object + if (reader) { + frame = reader->GetFrame(frame_position); + } + } + catch (const ReaderClosed & e) { } + catch (const OutOfBoundsFrame & e) { } - // Determine if any samples need to be copied - if (number_to_copy > 0) - { - // Debug - ZmqLogger::Instance()->AppendDebugMethod("AudioReaderSource::getNextAudioBlock", "number_to_copy", number_to_copy, "buffer_samples", buffer_samples, "buffer_channels", buffer_channels, "info.numSamples", info.numSamples, "speed", speed, "position", position); + // Get audio samples + if (reader && frame) { + if (sample_position + remaining_samples <= frame->GetAudioSamplesCount()) { + // Success, we have enough samples + for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++) { + if (channel < info.buffer->getNumChannels()) { + info.buffer->addFrom(channel, remaining_position, *frame->GetAudioSampleBuffer(), + channel, sample_position, remaining_samples); + } + } + sample_position += remaining_samples; + remaining_position += remaining_samples; + remaining_samples = 0; - // Loop through each channel and copy some samples - for (int channel = 0; channel < buffer_channels; channel++) - info.buffer->copyFrom(channel, info.startSample, *buffer, channel, position, number_to_copy); + } else if (sample_position + remaining_samples > frame->GetAudioSamplesCount()) { + // Not enough samples, take what we can + int amount_to_copy = frame->GetAudioSamplesCount() - sample_position; + for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++) { + if (channel < info.buffer->getNumChannels()) { + info.buffer->addFrom(channel, remaining_position, *frame->GetAudioSampleBuffer(), channel, + sample_position, amount_to_copy); + } + } + sample_position += amount_to_copy; + remaining_position += amount_to_copy; + remaining_samples -= amount_to_copy; + } - // Update the position of this audio source - position += number_to_copy; + // Increment frame position (if samples are all used up) + if (sample_position == frame->GetAudioSamplesCount()) { + frame_position += speed; + sample_position = 0; // reset for new frame + } + + } } - - // Adjust estimate frame number (the estimated frame number that is being played) - estimated_samples_per_frame = Frame::GetSamplesPerFrame(estimated_frame, reader->info.fps, reader->info.sample_rate, buffer_channels); - estimated_frame += double(info.numSamples) / double(estimated_samples_per_frame); } } @@ -233,21 +98,6 @@ void AudioReaderSource::prepareToPlay(int, double) {} // Release all resources void AudioReaderSource::releaseResources() { } -// Set the next read position of this source -void AudioReaderSource::setNextReadPosition (juce::int64 newPosition) -{ - // set position (if the new position is in range) - if (newPosition >= 0 && newPosition < buffer->getNumSamples()) - position = newPosition; -} - -// Get the next read position of this source -juce::int64 AudioReaderSource::getNextReadPosition() const -{ - // return the next read position - return position; -} - // Get the total length (in samples) of this audio source juce::int64 AudioReaderSource::getTotalLength() const { @@ -257,24 +107,3 @@ juce::int64 AudioReaderSource::getTotalLength() const else return 0; } - -// Determines if this audio source should repeat when it reaches the end -bool AudioReaderSource::isLooping() const -{ - // return if this source is looping - return repeat; -} - -// Set if this audio source should repeat when it reaches the end -void AudioReaderSource::setLooping (bool shouldLoop) -{ - // Set the repeat flag - repeat = shouldLoop; -} - -// Update the internal buffer used by this source -void AudioReaderSource::setBuffer (juce::AudioBuffer *audio_buffer) -{ - buffer = audio_buffer; - setNextReadPosition(0); -} diff --git a/src/AudioReaderSource.h b/src/AudioReaderSource.h index 81301f4e..c8b21e0f 100644 --- a/src/AudioReaderSource.h +++ b/src/AudioReaderSource.h @@ -31,33 +31,21 @@ namespace openshot class AudioReaderSource : public juce::PositionableAudioSource { private: - int position; /// The position of the audio source (index of buffer) - bool repeat; /// Repeat the audio source when finished - int size; /// The size of the internal buffer - juce::AudioBuffer *buffer; /// The audio sample buffer - int speed; /// The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) + int stream_position; /// The absolute stream position (required by PositionableAudioSource, but ignored) + int frame_position; /// The frame position (current frame for audio playback) + int speed; /// The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) ReaderBase *reader; /// The reader to pull samples from - int64_t frame_number; /// The current frame number std::shared_ptr frame; /// The current frame object that is being read - int64_t frame_position; /// The position of the current frame's buffer - double estimated_frame; /// The estimated frame position of the currently playing buffer - int estimated_samples_per_frame; /// The estimated samples per frame of video + int64_t sample_position; /// The position of the current frame's audio buffer openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking) - /// Get more samples from the reader - void GetMoreSamplesFromReader(); - - /// Reverse an audio buffer (for backwards audio) - juce::AudioBuffer* reverse_buffer(juce::AudioBuffer* buffer); - public: /// @brief Constructor that reads samples from a reader /// @param audio_reader This reader provides constant samples from a ReaderBase derived class /// @param starting_frame_number This is the frame number to start reading samples from the reader. - /// @param buffer_size The max number of samples to keep in the buffer at one time. - AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number, int buffer_size); + AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number); /// Destructor ~AudioReaderSource(); @@ -74,32 +62,24 @@ namespace openshot /// @brief Set the next read position of this source /// @param newPosition The sample # to start reading from - void setNextReadPosition (juce::int64 newPosition); + void setNextReadPosition (juce::int64 newPosition) { stream_position = newPosition; }; /// Get the next read position of this source - juce::int64 getNextReadPosition() const; + juce::int64 getNextReadPosition() const { return stream_position; }; /// Get the total length (in samples) of this audio source juce::int64 getTotalLength() const; - /// Determines if this audio source should repeat when it reaches the end - bool isLooping() const; + /// Looping is not support in OpenShot audio playback (this is always false) + bool isLooping() const { return false; }; - /// @brief Set if this audio source should repeat when it reaches the end + /// @brief This method is ignored (we do not support looping audio playback) /// @param shouldLoop Determines if the audio source should repeat when it reaches the end - void setLooping (bool shouldLoop); - - /// Update the internal buffer used by this source - void setBuffer (juce::AudioBuffer *audio_buffer); - - const ReaderInfo & getReaderInfo() const { return reader->info; } + void setLooping (bool shouldLoop) { }; /// Return the current frame object std::shared_ptr getFrame() const { return frame; } - /// Get the estimate frame that is playing at this moment - int64_t getEstimatedFrame() const { return int64_t(estimated_frame); } - /// Set Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) void setSpeed(int new_speed) { speed = new_speed; } /// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...) @@ -114,7 +94,7 @@ namespace openshot ReaderBase* Reader() const { return reader; } /// Seek to a specific frame - void Seek(int64_t new_position) { frame_number = new_position; estimated_frame = new_position; } + void Seek(int64_t new_position) { frame_position = new_position; sample_position=0; } }; diff --git a/src/Qt/AudioPlaybackThread.cpp b/src/Qt/AudioPlaybackThread.cpp index d1654458..7a83f636 100644 --- a/src/Qt/AudioPlaybackThread.cpp +++ b/src/Qt/AudioPlaybackThread.cpp @@ -27,12 +27,17 @@ using namespace juce; namespace openshot { - // Global reference to device manager AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::m_pInstance = NULL; - // Create or Get an instance of the device manager singleton - AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance() + // Create or Get audio device singleton with default settings (44100, 2) + AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance() + { + return AudioDeviceManagerSingleton::Instance(44100, 2); + } + + // Create or Get an instance of the device manager singleton (with custom sample rate & channels) + AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance(int rate, int channels) { if (!m_pInstance) { // Create the actual instance of device manager only once @@ -62,13 +67,20 @@ namespace openshot if (!selected_type.isEmpty()) m_pInstance->audioDeviceManager.setCurrentAudioDeviceType(selected_type, true); - // Initialize audio device only 1 time + // Settings for audio device playback + AudioDeviceManager::AudioDeviceSetup deviceSetup = AudioDeviceManager::AudioDeviceSetup(); + deviceSetup.sampleRate = rate; + deviceSetup.inputChannels = channels; + deviceSetup.outputChannels = channels; + + // Initialize audio device only 1 time juce::String audio_error = m_pInstance->audioDeviceManager.initialise ( 0, // number of input channels 2, // number of output channels nullptr, // no XML settings.. true, // select default device on failure - selected_device // preferredDefaultDeviceName + selected_device, // preferredDefaultDeviceName + &deviceSetup // sample_rate & channels ); // Persist any errors detected @@ -99,8 +111,7 @@ namespace openshot , mixer() , source(NULL) , sampleRate(0.0) - , numChannels(0) - , buffer_size(7000) + , numChannels(0) , is_playing(false) , time_thread("audio-buffer") , videoCache(cache) @@ -118,8 +129,8 @@ namespace openshot source->Reader(reader); else { // Create new audio source reader - source = new AudioReaderSource(reader, 1, buffer_size); - source->setLooping(true); // prevent this source from terminating when it reaches the end + auto starting_frame = 1; + source = new AudioReaderSource(reader, starting_frame); } // Set local vars @@ -166,8 +177,10 @@ namespace openshot if (source && !transport.isPlaying() && is_playing) { // Start new audio device (or get existing one) - // Add callback - AudioDeviceManagerSingleton::Instance()->audioDeviceManager.addAudioCallback(&player); + AudioDeviceManagerSingleton *audioInstance = AudioDeviceManagerSingleton::Instance(sampleRate, + numChannels); + // Add callback + audioInstance->audioDeviceManager.addAudioCallback(&player); // Create TimeSliceThread for audio buffering time_thread.startThread(); @@ -175,10 +188,10 @@ namespace openshot // Connect source to transport transport.setSource( source, - buffer_size, // tells it to buffer this many samples ahead + 0, // No read ahead buffer &time_thread, - sampleRate, - numChannels); + 0, // Sample rate correction (none) + numChannels); // max channels transport.setPosition(0); transport.setGain(1.0); @@ -200,7 +213,7 @@ namespace openshot transport.setSource(NULL); player.setSource(NULL); - AudioDeviceManagerSingleton::Instance()->audioDeviceManager.removeAudioCallback(&player); + audioInstance->audioDeviceManager.removeAudioCallback(&player); // Remove source delete source; diff --git a/src/Qt/AudioPlaybackThread.h b/src/Qt/AudioPlaybackThread.h index 821b7138..d7f623d1 100644 --- a/src/Qt/AudioPlaybackThread.h +++ b/src/Qt/AudioPlaybackThread.h @@ -44,14 +44,18 @@ private: AudioDeviceManagerSingleton(){ initialise_error=""; }; /// Private variable to keep track of singleton instance - static AudioDeviceManagerSingleton * m_pInstance; + static AudioDeviceManagerSingleton* m_pInstance; public: /// Error found during JUCE initialise method std::string initialise_error; - /// Override with no channels and no preferred audio device - static AudioDeviceManagerSingleton * Instance(); + /// Override with default sample rate & channels (44100, 2) and no preferred audio device + static AudioDeviceManagerSingleton* Instance(); + + /// Override with custom sample rate & channels and no preferred audio device + /// sample rate and channels are only set on 1st call (when singleton is created) + static AudioDeviceManagerSingleton* Instance(int rate, int channels); /// Public device manager property juce::AudioDeviceManager audioDeviceManager; @@ -72,8 +76,6 @@ public: double sampleRate; int numChannels; juce::WaitableEvent play; - juce::WaitableEvent played; - int buffer_size; bool is_playing; juce::TimeSliceThread time_thread; openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking) @@ -86,9 +88,6 @@ public: /// Set the current thread's reader void Reader(openshot::ReaderBase *reader); - /// Return the current audio transport buffer size (to determine latency) - int getBufferSize() { return buffer_size; } - /// Get the current frame object (which is filling the buffer) std::shared_ptr getFrame(); diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index cf63c8e0..75c325d5 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -62,11 +62,6 @@ namespace openshot using micro_sec = std::chrono::microseconds; using double_micro_sec = std::chrono::duration; - // Calculate latency of audio thread (i.e. how many microseconds before samples are audible) - // TODO: This is the experimental amount of latency I have on my system audio playback - //const auto audio_latency = double_micro_sec(1000000.0 * (audioPlayback->getBufferSize() / reader->info.sample_rate)); - const auto audio_latency = double_micro_sec(240000); - // Init start_time of playback std::chrono::time_point start_time; start_time = std::chrono::time_point_cast(std::chrono::high_resolution_clock::now()); ///< timestamp playback starts @@ -102,7 +97,7 @@ namespace openshot // Calculate the diff between 'now' and the predicted frame end time const auto current_time = std::chrono::high_resolution_clock::now(); - const auto remaining_time = double_micro_sec(start_time + audio_latency + + const auto remaining_time = double_micro_sec(start_time + (frame_duration * playback_frames) - current_time); // Sleep to display video image on screen diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index f1ddf42a..b2288866 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -28,7 +28,7 @@ namespace openshot VideoCacheThread::VideoCacheThread() : Thread("video-cache"), speed(1), is_playing(false), reader(NULL), current_display_frame(1), cached_frame_count(0), - min_frames_ahead(12), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6) + min_frames_ahead(24), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6) { }