openshot/libopenshot
0
0
Fork 0
mirror of https://github.com/OpenShot/libopenshot.git synced 2026-03-02 08:53:52 -08:00
Code Issues Packages Projects Releases Wiki Activity

Big refactor of AudioReaderSource, to be simpler, and only fill the audio samples requested (removed double/internal buffer complexity). Also, we now initialize the audio device manager at a specific sampleRate and # of channels correctly. Removed 'latency' adjustment in PrivatePlayer (no longer needed with no read-ahead buffer). Increased the min_frames_required on the video cache to 24.

Browse Source
This commit is contained in:
Jonathan Thomas
2022-01-31 15:18:07 -06:00
parent 133bae40c3
commit 176f2fff9c
6 changed files with 99 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

271
src/AudioReaderSource.cpp
View File

@@ -13,217 +13,82 @@
#include "AudioReaderSource.h"
#include "Exceptions.h"
#include "Frame.h"
#include "ZmqLogger.h"
using namespace std;
using namespace openshot;
// Constructor that reads samples from a reader
AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number, int buffer_size)
: reader(audio_reader), frame_number(starting_frame_number), videoCache(NULL),
size(buffer_size), position(0), frame_position(0), estimated_frame(0), speed(1) {
// Initialize an audio buffer (based on reader)
buffer = new juce::AudioBuffer<float>(reader->info.channels, size);
// initialize the audio samples to zero (silence)
buffer->clear();
AudioReaderSource::AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number)
: reader(audio_reader), frame_position(starting_frame_number), videoCache(NULL), frame(NULL),
sample_position(0), speed(1), stream_position(0) {
}
// Destructor
AudioReaderSource::~AudioReaderSource()
{
delete buffer;
buffer = NULL;
}
// Get more samples from the reader
void AudioReaderSource::GetMoreSamplesFromReader()
{
// Determine the amount of samples needed to fill up this buffer
int amount_needed = position; // replace these used samples
int amount_remaining = size - amount_needed; // these are unused samples, and need to be carried forward
if (!frame) {
// If no frame, load entire buffer
amount_needed = size;
amount_remaining = 0;
}
// Debug
ZmqLogger::Instance()->AppendDebugMethod("AudioReaderSource::GetMoreSamplesFromReader", "amount_needed", amount_needed, "amount_remaining", amount_remaining);
// Init estimated buffer equal to the current frame position (before getting more samples)
estimated_frame = frame_number;
// Init new buffer
auto *new_buffer = new juce::AudioBuffer<float>(reader->info.channels, size);
new_buffer->clear();
// Move the remaining samples into new buffer (if any)
if (amount_remaining > 0) {
for (int channel = 0; channel < buffer->getNumChannels(); channel++)
new_buffer->addFrom(channel, 0, *buffer, channel, position, amount_remaining);
position = amount_remaining;
} else
// reset position to 0
position = 0;
// Loop through frames until buffer filled
while (amount_needed > 0 && speed == 1 && frame_number >= 1 && frame_number <= reader->info.video_length) {
// Get the next frame (if position is zero)
if (frame_position == 0) {
try {
// Get frame object
frame = reader->GetFrame(frame_number);
frame_number = frame_number + speed;
} catch (const ReaderClosed & e) {
break;
} catch (const OutOfBoundsFrame & e) {
break;
}
}
bool frame_completed = false;
int amount_to_copy = 0;
if (frame)
amount_to_copy = frame->GetAudioSamplesCount() - frame_position;
if (amount_to_copy > amount_needed) {
// Don't copy too many samples (we don't want to overflow the buffer)
amount_to_copy = amount_needed;
amount_needed = 0;
} else {
// Not enough to fill the buffer (so use the entire frame)
amount_needed -= amount_to_copy;
frame_completed = true;
}
// Load all of its samples into the buffer
if (frame)
for (int channel = 0; channel < new_buffer->getNumChannels(); channel++)
new_buffer->addFrom(channel, position, *frame->GetAudioSampleBuffer(), channel, frame_position, amount_to_copy);
// Adjust remaining samples
position += amount_to_copy;
if (frame_completed)
// Reset frame buffer position (which will load a new frame on the next loop)
frame_position = 0;
else
// Continue tracking the current frame's position
frame_position += amount_to_copy;
}
// Delete old buffer
buffer->clear();
delete buffer;
// Replace buffer and reset position
buffer = new_buffer;
position = 0;
}
// Reverse an audio buffer
juce::AudioBuffer<float>* AudioReaderSource::reverse_buffer(juce::AudioBuffer<float>* buffer)
{
int number_of_samples = buffer->getNumSamples();
int channels = buffer->getNumChannels();
// Debug
ZmqLogger::Instance()->AppendDebugMethod("AudioReaderSource::reverse_buffer", "number_of_samples", number_of_samples, "channels", channels);
// Reverse array (create new buffer to hold the reversed version)
auto *reversed = new juce::AudioBuffer<float>(channels, number_of_samples);
reversed->clear();
for (int channel = 0; channel < channels; channel++)
{
int n=0;
for (int s = number_of_samples - 1; s >= 0; s--, n++)
reversed->getWritePointer(channel)[n] = buffer->getWritePointer(channel)[s];
}
// Copy the samples back to the original array
buffer->clear();
// Loop through channels, and get audio samples
for (int channel = 0; channel < channels; channel++)
// Get the audio samples for this channel
buffer->addFrom(channel, 0, reversed->getReadPointer(channel), number_of_samples, 1.0f);
delete reversed;
reversed = NULL;
// return pointer or passed in object (so this method can be chained together)
return buffer;
}
// Get the next block of audio samples
void AudioReaderSource::getNextAudioBlock(const juce::AudioSourceChannelInfo& info)
{
int buffer_samples = buffer->getNumSamples();
int buffer_channels = buffer->getNumChannels();
if (info.numSamples > 0) {
int number_to_copy = 0;
int remaining_samples = info.numSamples;
int remaining_position = info.startSample;
// Do we need more samples?
if (speed == 1 && videoCache->isReady()) {
// Only refill buffers if speed is normal
if ((reader && reader->IsOpen() && !frame) or
(reader && reader->IsOpen() && buffer_samples - position < info.numSamples))
// Refill buffer from reader
GetMoreSamplesFromReader();
} else {
// Fill buffer with silence and clear current frame
// Pause and fill buffer with silence (wait for pre-roll)
if (speed != 1 || !videoCache->isReady()) {
info.buffer->clear();
// Empty internal buffer also
buffer->clear();
position = 0;
return;
}
// Determine how many samples to copy
if (position + info.numSamples <= buffer_samples)
{
// copy the full amount requested
number_to_copy = info.numSamples;
}
else if (position > buffer_samples)
{
// copy nothing
number_to_copy = 0;
}
else if (buffer_samples - position > 0)
{
// only copy what is left in the buffer
number_to_copy = buffer_samples - position;
}
else
{
// copy nothing
number_to_copy = 0;
}
while (remaining_samples > 0) {
try {
// Get current frame object
if (reader) {
frame = reader->GetFrame(frame_position);
}
}
catch (const ReaderClosed & e) { }
catch (const OutOfBoundsFrame & e) { }
// Determine if any samples need to be copied
if (number_to_copy > 0)
{
// Debug
ZmqLogger::Instance()->AppendDebugMethod("AudioReaderSource::getNextAudioBlock", "number_to_copy", number_to_copy, "buffer_samples", buffer_samples, "buffer_channels", buffer_channels, "info.numSamples", info.numSamples, "speed", speed, "position", position);
// Get audio samples
if (reader && frame) {
if (sample_position + remaining_samples <= frame->GetAudioSamplesCount()) {
// Success, we have enough samples
for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++) {
if (channel < info.buffer->getNumChannels()) {
info.buffer->addFrom(channel, remaining_position, *frame->GetAudioSampleBuffer(),
channel, sample_position, remaining_samples);
}
}
sample_position += remaining_samples;
remaining_position += remaining_samples;
remaining_samples = 0;
// Loop through each channel and copy some samples
for (int channel = 0; channel < buffer_channels; channel++)
info.buffer->copyFrom(channel, info.startSample, *buffer, channel, position, number_to_copy);
} else if (sample_position + remaining_samples > frame->GetAudioSamplesCount()) {
// Not enough samples, take what we can
int amount_to_copy = frame->GetAudioSamplesCount() - sample_position;
for (int channel = 0; channel < frame->GetAudioChannelsCount(); channel++) {
if (channel < info.buffer->getNumChannels()) {
info.buffer->addFrom(channel, remaining_position, *frame->GetAudioSampleBuffer(), channel,
sample_position, amount_to_copy);
}
}
sample_position += amount_to_copy;
remaining_position += amount_to_copy;
remaining_samples -= amount_to_copy;
}
// Update the position of this audio source
position += number_to_copy;
// Increment frame position (if samples are all used up)
if (sample_position == frame->GetAudioSamplesCount()) {
frame_position += speed;
sample_position = 0; // reset for new frame
}
}
}
// Adjust estimate frame number (the estimated frame number that is being played)
estimated_samples_per_frame = Frame::GetSamplesPerFrame(estimated_frame, reader->info.fps, reader->info.sample_rate, buffer_channels);
estimated_frame += double(info.numSamples) / double(estimated_samples_per_frame);
}
}
@@ -233,21 +98,6 @@ void AudioReaderSource::prepareToPlay(int, double) {}
// Release all resources
void AudioReaderSource::releaseResources() { }
// Set the next read position of this source
void AudioReaderSource::setNextReadPosition (juce::int64 newPosition)
{
// set position (if the new position is in range)
if (newPosition >= 0 && newPosition < buffer->getNumSamples())
position = newPosition;
}
// Get the next read position of this source
juce::int64 AudioReaderSource::getNextReadPosition() const
{
// return the next read position
return position;
}
// Get the total length (in samples) of this audio source
juce::int64 AudioReaderSource::getTotalLength() const
{
@@ -257,24 +107,3 @@ juce::int64 AudioReaderSource::getTotalLength() const
else
return 0;
}
// Determines if this audio source should repeat when it reaches the end
bool AudioReaderSource::isLooping() const
{
// return if this source is looping
return repeat;
}
// Set if this audio source should repeat when it reaches the end
void AudioReaderSource::setLooping (bool shouldLoop)
{
// Set the repeat flag
repeat = shouldLoop;
}
// Update the internal buffer used by this source
void AudioReaderSource::setBuffer (juce::AudioBuffer<float> *audio_buffer)
{
buffer = audio_buffer;
setNextReadPosition(0);
}

44
src/AudioReaderSource.h
View File

@@ -31,33 +31,21 @@ namespace openshot
class AudioReaderSource : public juce::PositionableAudioSource
{
private:
int position; /// The position of the audio source (index of buffer)
bool repeat; /// Repeat the audio source when finished
int size; /// The size of the internal buffer
juce::AudioBuffer<float> *buffer; /// The audio sample buffer
int speed; /// The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
int stream_position; /// The absolute stream position (required by PositionableAudioSource, but ignored)
int frame_position; /// The frame position (current frame for audio playback)
int speed; /// The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
ReaderBase *reader; /// The reader to pull samples from
int64_t frame_number; /// The current frame number
std::shared_ptr<Frame> frame; /// The current frame object that is being read
int64_t frame_position; /// The position of the current frame's buffer
double estimated_frame; /// The estimated frame position of the currently playing buffer
int estimated_samples_per_frame; /// The estimated samples per frame of video
int64_t sample_position; /// The position of the current frame's audio buffer
openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking)
/// Get more samples from the reader
void GetMoreSamplesFromReader();
/// Reverse an audio buffer (for backwards audio)
juce::AudioBuffer<float>* reverse_buffer(juce::AudioBuffer<float>* buffer);
public:
/// @brief Constructor that reads samples from a reader
/// @param audio_reader This reader provides constant samples from a ReaderBase derived class
/// @param starting_frame_number This is the frame number to start reading samples from the reader.
/// @param buffer_size The max number of samples to keep in the buffer at one time.
AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number, int buffer_size);
AudioReaderSource(ReaderBase *audio_reader, int64_t starting_frame_number);
/// Destructor
~AudioReaderSource();
@@ -74,32 +62,24 @@ namespace openshot
/// @brief Set the next read position of this source
/// @param newPosition The sample # to start reading from
void setNextReadPosition (juce::int64 newPosition);
void setNextReadPosition (juce::int64 newPosition) { stream_position = newPosition; };
/// Get the next read position of this source
juce::int64 getNextReadPosition() const;
juce::int64 getNextReadPosition() const { return stream_position; };
/// Get the total length (in samples) of this audio source
juce::int64 getTotalLength() const;
/// Determines if this audio source should repeat when it reaches the end
bool isLooping() const;
/// Looping is not support in OpenShot audio playback (this is always false)
bool isLooping() const { return false; };
/// @brief Set if this audio source should repeat when it reaches the end
/// @brief This method is ignored (we do not support looping audio playback)
/// @param shouldLoop Determines if the audio source should repeat when it reaches the end
void setLooping (bool shouldLoop);
/// Update the internal buffer used by this source
void setBuffer (juce::AudioBuffer<float> *audio_buffer);
const ReaderInfo & getReaderInfo() const { return reader->info; }
void setLooping (bool shouldLoop) { };
/// Return the current frame object
std::shared_ptr<Frame> getFrame() const { return frame; }
/// Get the estimate frame that is playing at this moment
int64_t getEstimatedFrame() const { return int64_t(estimated_frame); }
/// Set Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
void setSpeed(int new_speed) { speed = new_speed; }
/// Get Speed (The speed and direction to playback a reader (1=normal, 2=fast, 3=faster, -1=rewind, etc...)
@@ -114,7 +94,7 @@ namespace openshot
ReaderBase* Reader() const { return reader; }
/// Seek to a specific frame
void Seek(int64_t new_position) { frame_number = new_position; estimated_frame = new_position; }
void Seek(int64_t new_position) { frame_position = new_position; sample_position=0; }
};

43
src/Qt/AudioPlaybackThread.cpp
View File

@@ -27,12 +27,17 @@ using namespace juce;
namespace openshot
{
// Global reference to device manager
AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::m_pInstance = NULL;
// Create or Get an instance of the device manager singleton
AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance()
// Create or Get audio device singleton with default settings (44100, 2)
AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance()
{
return AudioDeviceManagerSingleton::Instance(44100, 2);
}
// Create or Get an instance of the device manager singleton (with custom sample rate & channels)
AudioDeviceManagerSingleton *AudioDeviceManagerSingleton::Instance(int rate, int channels)
{
if (!m_pInstance) {
// Create the actual instance of device manager only once
@@ -62,13 +67,20 @@ namespace openshot
if (!selected_type.isEmpty())
m_pInstance->audioDeviceManager.setCurrentAudioDeviceType(selected_type, true);
// Initialize audio device only 1 time
// Settings for audio device playback
AudioDeviceManager::AudioDeviceSetup deviceSetup = AudioDeviceManager::AudioDeviceSetup();
deviceSetup.sampleRate = rate;
deviceSetup.inputChannels = channels;
deviceSetup.outputChannels = channels;
// Initialize audio device only 1 time
juce::String audio_error = m_pInstance->audioDeviceManager.initialise (
0, // number of input channels
2, // number of output channels
nullptr, // no XML settings..
true, // select default device on failure
selected_device // preferredDefaultDeviceName
selected_device, // preferredDefaultDeviceName
&deviceSetup // sample_rate & channels
);
// Persist any errors detected
@@ -99,8 +111,7 @@ namespace openshot
, mixer()
, source(NULL)
, sampleRate(0.0)
, numChannels(0)
, buffer_size(7000)
, numChannels(0)
, is_playing(false)
, time_thread("audio-buffer")
, videoCache(cache)
@@ -118,8 +129,8 @@ namespace openshot
source->Reader(reader);
else {
// Create new audio source reader
source = new AudioReaderSource(reader, 1, buffer_size);
source->setLooping(true); // prevent this source from terminating when it reaches the end
auto starting_frame = 1;
source = new AudioReaderSource(reader, starting_frame);
}
// Set local vars
@@ -166,8 +177,10 @@ namespace openshot
if (source && !transport.isPlaying() && is_playing) {
// Start new audio device (or get existing one)
// Add callback
AudioDeviceManagerSingleton::Instance()->audioDeviceManager.addAudioCallback(&player);
AudioDeviceManagerSingleton *audioInstance = AudioDeviceManagerSingleton::Instance(sampleRate,
numChannels);
// Add callback
audioInstance->audioDeviceManager.addAudioCallback(&player);
// Create TimeSliceThread for audio buffering
time_thread.startThread();
@@ -175,10 +188,10 @@ namespace openshot
// Connect source to transport
transport.setSource(
source,
buffer_size, // tells it to buffer this many samples ahead
0, // No read ahead buffer
&time_thread,
sampleRate,
numChannels);
0, // Sample rate correction (none)
numChannels); // max channels
transport.setPosition(0);
transport.setGain(1.0);
@@ -200,7 +213,7 @@ namespace openshot
transport.setSource(NULL);
player.setSource(NULL);
AudioDeviceManagerSingleton::Instance()->audioDeviceManager.removeAudioCallback(&player);
audioInstance->audioDeviceManager.removeAudioCallback(&player);
// Remove source
delete source;

15
src/Qt/AudioPlaybackThread.h
View File

@@ -44,14 +44,18 @@ private:
AudioDeviceManagerSingleton(){ initialise_error=""; };
/// Private variable to keep track of singleton instance
static AudioDeviceManagerSingleton * m_pInstance;
static AudioDeviceManagerSingleton* m_pInstance;
public:
/// Error found during JUCE initialise method
std::string initialise_error;
/// Override with no channels and no preferred audio device
static AudioDeviceManagerSingleton * Instance();
/// Override with default sample rate & channels (44100, 2) and no preferred audio device
static AudioDeviceManagerSingleton* Instance();
/// Override with custom sample rate & channels and no preferred audio device
/// sample rate and channels are only set on 1st call (when singleton is created)
static AudioDeviceManagerSingleton* Instance(int rate, int channels);
/// Public device manager property
juce::AudioDeviceManager audioDeviceManager;
@@ -72,8 +76,6 @@ public:
double sampleRate;
int numChannels;
juce::WaitableEvent play;
juce::WaitableEvent played;
int buffer_size;
bool is_playing;
juce::TimeSliceThread time_thread;
openshot::VideoCacheThread *videoCache; /// The cache thread (for pre-roll checking)
@@ -86,9 +88,6 @@ public:
/// Set the current thread's reader
void Reader(openshot::ReaderBase *reader);
/// Return the current audio transport buffer size (to determine latency)
int getBufferSize() { return buffer_size; }
/// Get the current frame object (which is filling the buffer)
std::shared_ptr<openshot::Frame> getFrame();

7
src/Qt/PlayerPrivate.cpp
View File

@@ -62,11 +62,6 @@ namespace openshot
using micro_sec = std::chrono::microseconds;
using double_micro_sec = std::chrono::duration<double, micro_sec::period>;
// Calculate latency of audio thread (i.e. how many microseconds before samples are audible)
// TODO: This is the experimental amount of latency I have on my system audio playback
//const auto audio_latency = double_micro_sec(1000000.0 * (audioPlayback->getBufferSize() / reader->info.sample_rate));
const auto audio_latency = double_micro_sec(240000);
// Init start_time of playback
std::chrono::time_point<std::chrono::system_clock, std::chrono::microseconds> start_time;
start_time = std::chrono::time_point_cast<micro_sec>(std::chrono::high_resolution_clock::now()); ///< timestamp playback starts
@@ -102,7 +97,7 @@ namespace openshot
// Calculate the diff between 'now' and the predicted frame end time
const auto current_time = std::chrono::high_resolution_clock::now();
const auto remaining_time = double_micro_sec(start_time + audio_latency +
const auto remaining_time = double_micro_sec(start_time +
(frame_duration * playback_frames) - current_time);
// Sleep to display video image on screen

2
src/Qt/VideoCacheThread.cpp
View File

@@ -28,7 +28,7 @@ namespace openshot
VideoCacheThread::VideoCacheThread()
: Thread("video-cache"), speed(1), is_playing(false),
reader(NULL), current_display_frame(1), cached_frame_count(0),
min_frames_ahead(12), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6)
min_frames_ahead(24), max_frames_ahead(OPEN_MP_NUM_PROCESSORS * 6)
{
}