From 6be754085298018ebbb91bf1c08416e32dd0f98b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 31 Dec 2015 04:17:54 -0600 Subject: [PATCH] Fixed a bug with time mapping for animations and videos without audio. --- src/Clip.cpp | 300 ++++++++++++++++++++------------------- src/Qt/PlayerPrivate.cpp | 124 ++++++++-------- src/QtPlayer.cpp | 2 - 3 files changed, 217 insertions(+), 209 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 95e103e3..4f98ad2b 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -358,169 +358,181 @@ tr1::shared_ptr Clip::get_time_mapped_frame(tr1::shared_ptr frame, int channels = reader->info.channels; int number_of_samples = GetOrCreateFrame(new_frame_number)->GetAudioSamplesCount(); - // Determine if we are speeding up or slowing down - if (time.GetRepeatFraction(frame_number).den > 1) - { - // SLOWING DOWN AUDIO - // Resample data, and return new buffer pointer - AudioSampleBuffer *resampled_buffer = NULL; - int resampled_buffer_size = 0; + // Only resample audio if needed + if (reader->info.has_audio) { + // Determine if we are speeding up or slowing down + if (time.GetRepeatFraction(frame_number).den > 1) { + // SLOWING DOWN AUDIO + // Resample data, and return new buffer pointer + AudioSampleBuffer *resampled_buffer = NULL; + int resampled_buffer_size = 0; - // SLOW DOWN audio (split audio) - samples = new juce::AudioSampleBuffer(channels, number_of_samples); - samples->clear(); - - // Loop through channels, and get audio samples - for (int channel = 0; channel < channels; channel++) - // Get the audio samples for this channel - samples->addFrom(channel, 0, GetOrCreateFrame(new_frame_number)->GetAudioSamples(channel), number_of_samples, 1.0f); - - // Reverse the samples (if needed) - if (!time.IsIncreasing(frame_number)) - reverse_buffer(samples); - - // Resample audio to be X times slower (where X is the denominator of the repeat fraction) - resampler->SetBuffer(samples, 1.0 / time.GetRepeatFraction(frame_number).den); - - // Resample the data (since it's the 1st slice) - resampled_buffer = resampler->GetResampledBuffer(); - - // Get the length of the resampled buffer (if one exists) - resampled_buffer_size = resampled_buffer->getNumSamples(); - - // Just take the samples we need for the requested frame - int start = (number_of_samples * (time.GetRepeatFraction(frame_number).num - 1)); - if (start > 0) - start -= 1; - for (int channel = 0; channel < channels; channel++) - // Add new (slower) samples, to the frame object - new_frame->AddAudio(true, channel, 0, resampled_buffer->getReadPointer(channel, start), number_of_samples, 1.0f); - - // Clean up - resampled_buffer = NULL; - - } - else if (abs(delta) > 1 && abs(delta) < 100) - { - int start = 0; - if (delta > 0) - { - // SPEED UP (multiple frames of audio), as long as it's not more than X frames - int total_delta_samples = 0; - for (int delta_frame = new_frame_number - (delta - 1); delta_frame <= new_frame_number; delta_frame++) - total_delta_samples += Frame::GetSamplesPerFrame(delta_frame, reader->info.fps, reader->info.sample_rate, reader->info.channels); - - // Allocate a new sample buffer for these delta frames - samples = new juce::AudioSampleBuffer(channels, total_delta_samples); + // SLOW DOWN audio (split audio) + samples = new juce::AudioSampleBuffer(channels, number_of_samples); samples->clear(); - // Loop through each frame in this delta - for (int delta_frame = new_frame_number - (delta - 1); delta_frame <= new_frame_number; delta_frame++) - { - // buffer to hold detal samples - int number_of_delta_samples = GetOrCreateFrame(delta_frame)->GetAudioSamplesCount(); - AudioSampleBuffer* delta_samples = new juce::AudioSampleBuffer(channels, number_of_delta_samples); - delta_samples->clear(); + // Loop through channels, and get audio samples + for (int channel = 0; channel < channels; channel++) + // Get the audio samples for this channel + samples->addFrom(channel, 0, GetOrCreateFrame(new_frame_number)->GetAudioSamples(channel), + number_of_samples, 1.0f); - for (int channel = 0; channel < channels; channel++) - delta_samples->addFrom(channel, 0, GetOrCreateFrame(delta_frame)->GetAudioSamples(channel), number_of_delta_samples, 1.0f); + // Reverse the samples (if needed) + if (!time.IsIncreasing(frame_number)) + reverse_buffer(samples); - // Reverse the samples (if needed) - if (!time.IsIncreasing(frame_number)) - reverse_buffer(delta_samples); + // Resample audio to be X times slower (where X is the denominator of the repeat fraction) + resampler->SetBuffer(samples, 1.0 / time.GetRepeatFraction(frame_number).den); - // Copy the samples to - for (int channel = 0; channel < channels; channel++) - // Get the audio samples for this channel - samples->addFrom(channel, start, delta_samples->getReadPointer(channel), number_of_delta_samples, 1.0f); + // Resample the data (since it's the 1st slice) + resampled_buffer = resampler->GetResampledBuffer(); - // Clean up - delete delta_samples; - delta_samples = NULL; + // Get the length of the resampled buffer (if one exists) + resampled_buffer_size = resampled_buffer->getNumSamples(); + + // Just take the samples we need for the requested frame + int start = (number_of_samples * (time.GetRepeatFraction(frame_number).num - 1)); + if (start > 0) + start -= 1; + for (int channel = 0; channel < channels; channel++) + // Add new (slower) samples, to the frame object + new_frame->AddAudio(true, channel, 0, resampled_buffer->getReadPointer(channel, start), + number_of_samples, 1.0f); + + // Clean up + resampled_buffer = NULL; - // Increment start position - start += number_of_delta_samples; - } } - else - { - // SPEED UP (multiple frames of audio), as long as it's not more than X frames - int total_delta_samples = 0; - for (int delta_frame = new_frame_number - (delta + 1); delta_frame >= new_frame_number; delta_frame--) - total_delta_samples += Frame::GetSamplesPerFrame(delta_frame, reader->info.fps, reader->info.sample_rate, reader->info.channels); + else if (abs(delta) > 1 && abs(delta) < 100) { + int start = 0; + if (delta > 0) { + // SPEED UP (multiple frames of audio), as long as it's not more than X frames + int total_delta_samples = 0; + for (int delta_frame = new_frame_number - (delta - 1); + delta_frame <= new_frame_number; delta_frame++) + total_delta_samples += Frame::GetSamplesPerFrame(delta_frame, reader->info.fps, + reader->info.sample_rate, + reader->info.channels); - // Allocate a new sample buffer for these delta frames - samples = new juce::AudioSampleBuffer(channels, total_delta_samples); + // Allocate a new sample buffer for these delta frames + samples = new juce::AudioSampleBuffer(channels, total_delta_samples); + samples->clear(); + + // Loop through each frame in this delta + for (int delta_frame = new_frame_number - (delta - 1); + delta_frame <= new_frame_number; delta_frame++) { + // buffer to hold detal samples + int number_of_delta_samples = GetOrCreateFrame(delta_frame)->GetAudioSamplesCount(); + AudioSampleBuffer *delta_samples = new juce::AudioSampleBuffer(channels, + number_of_delta_samples); + delta_samples->clear(); + + for (int channel = 0; channel < channels; channel++) + delta_samples->addFrom(channel, 0, GetOrCreateFrame(delta_frame)->GetAudioSamples(channel), + number_of_delta_samples, 1.0f); + + // Reverse the samples (if needed) + if (!time.IsIncreasing(frame_number)) + reverse_buffer(delta_samples); + + // Copy the samples to + for (int channel = 0; channel < channels; channel++) + // Get the audio samples for this channel + samples->addFrom(channel, start, delta_samples->getReadPointer(channel), + number_of_delta_samples, 1.0f); + + // Clean up + delete delta_samples; + delta_samples = NULL; + + // Increment start position + start += number_of_delta_samples; + } + } + else { + // SPEED UP (multiple frames of audio), as long as it's not more than X frames + int total_delta_samples = 0; + for (int delta_frame = new_frame_number - (delta + 1); + delta_frame >= new_frame_number; delta_frame--) + total_delta_samples += Frame::GetSamplesPerFrame(delta_frame, reader->info.fps, + reader->info.sample_rate, + reader->info.channels); + + // Allocate a new sample buffer for these delta frames + samples = new juce::AudioSampleBuffer(channels, total_delta_samples); + samples->clear(); + + // Loop through each frame in this delta + for (int delta_frame = new_frame_number - (delta + 1); + delta_frame >= new_frame_number; delta_frame--) { + // buffer to hold delta samples + int number_of_delta_samples = GetOrCreateFrame(delta_frame)->GetAudioSamplesCount(); + AudioSampleBuffer *delta_samples = new juce::AudioSampleBuffer(channels, + number_of_delta_samples); + delta_samples->clear(); + + for (int channel = 0; channel < channels; channel++) + delta_samples->addFrom(channel, 0, GetOrCreateFrame(delta_frame)->GetAudioSamples(channel), + number_of_delta_samples, 1.0f); + + // Reverse the samples (if needed) + if (!time.IsIncreasing(frame_number)) + reverse_buffer(delta_samples); + + // Copy the samples to + for (int channel = 0; channel < channels; channel++) + // Get the audio samples for this channel + samples->addFrom(channel, start, delta_samples->getReadPointer(channel), + number_of_delta_samples, 1.0f); + + // Clean up + delete delta_samples; + delta_samples = NULL; + + // Increment start position + start += number_of_delta_samples; + } + } + + // Resample audio to be X times faster (where X is the delta of the repeat fraction) + resampler->SetBuffer(samples, float(start) / float(number_of_samples)); + + // Resample data, and return new buffer pointer + AudioSampleBuffer *buffer = resampler->GetResampledBuffer(); + int resampled_buffer_size = buffer->getNumSamples(); + + // Add the newly resized audio samples to the current frame + for (int channel = 0; channel < channels; channel++) + // Add new (slower) samples, to the frame object + new_frame->AddAudio(true, channel, 0, buffer->getReadPointer(channel), number_of_samples, 1.0f); + + // Clean up + buffer = NULL; + } + else { + // Use the samples on this frame (but maybe reverse them if needed) + samples = new juce::AudioSampleBuffer(channels, number_of_samples); samples->clear(); - // Loop through each frame in this delta - for (int delta_frame = new_frame_number - (delta + 1); delta_frame >= new_frame_number; delta_frame--) - { - // buffer to hold delta samples - int number_of_delta_samples = GetOrCreateFrame(delta_frame)->GetAudioSamplesCount(); - AudioSampleBuffer* delta_samples = new juce::AudioSampleBuffer(channels, number_of_delta_samples); - delta_samples->clear(); + // Loop through channels, and get audio samples + for (int channel = 0; channel < channels; channel++) + // Get the audio samples for this channel + samples->addFrom(channel, 0, frame->GetAudioSamples(channel), number_of_samples, 1.0f); - for (int channel = 0; channel < channels; channel++) - delta_samples->addFrom(channel, 0, GetOrCreateFrame(delta_frame)->GetAudioSamples(channel), number_of_delta_samples, 1.0f); + // reverse the samples + if (!time.IsIncreasing(frame_number)) + reverse_buffer(samples); - // Reverse the samples (if needed) - if (!time.IsIncreasing(frame_number)) - reverse_buffer(delta_samples); + // Add reversed samples to the frame object + for (int channel = 0; channel < channels; channel++) + new_frame->AddAudio(true, channel, 0, samples->getReadPointer(channel), number_of_samples, 1.0f); - // Copy the samples to - for (int channel = 0; channel < channels; channel++) - // Get the audio samples for this channel - samples->addFrom(channel, start, delta_samples->getReadPointer(channel), number_of_delta_samples, 1.0f); - // Clean up - delete delta_samples; - delta_samples = NULL; - - // Increment start position - start += number_of_delta_samples; - } } - // Resample audio to be X times faster (where X is the delta of the repeat fraction) - resampler->SetBuffer(samples, float(start) / float(number_of_samples)); - - // Resample data, and return new buffer pointer - AudioSampleBuffer *buffer = resampler->GetResampledBuffer(); - int resampled_buffer_size = buffer->getNumSamples(); - - // Add the newly resized audio samples to the current frame - for (int channel = 0; channel < channels; channel++) - // Add new (slower) samples, to the frame object - new_frame->AddAudio(true, channel, 0, buffer->getReadPointer(channel), number_of_samples, 1.0f); - - // Clean up - buffer = NULL; + delete samples; + samples = NULL; } - else - { - // Use the samples on this frame (but maybe reverse them if needed) - samples = new juce::AudioSampleBuffer(channels, number_of_samples); - samples->clear(); - - // Loop through channels, and get audio samples - for (int channel = 0; channel < channels; channel++) - // Get the audio samples for this channel - samples->addFrom(channel, 0, frame->GetAudioSamples(channel), number_of_samples, 1.0f); - - // reverse the samples - if (!time.IsIncreasing(frame_number)) - reverse_buffer(samples); - - // Add reversed samples to the frame object - for (int channel = 0; channel < channels; channel++) - new_frame->AddAudio(true, channel, 0, samples->getReadPointer(channel), number_of_samples, 1.0f); - - - } - - delete samples; - samples = NULL; // Return new time mapped frame return new_frame; diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index db788ed3..2e78670a 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -51,87 +51,85 @@ namespace openshot // Start thread void PlayerPrivate::run() { - // bail if no reader set - if (!reader) - return; + // bail if no reader set + if (!reader) + return; - // Start the threads - if (reader->info.has_audio) - audioPlayback->startThread(8); - if (reader->info.has_video) { - videoCache->startThread(2); - videoPlayback->startThread(4); - } + // Start the threads + if (reader->info.has_audio) + audioPlayback->startThread(8); + if (reader->info.has_video) { + videoCache->startThread(2); + videoPlayback->startThread(4); + } - while (!threadShouldExit()) { + while (!threadShouldExit()) { - // Calculate the milliseconds a single frame should stay on the screen - double frame_time = (1000.0 / reader->info.fps.ToDouble()); + // Calculate the milliseconds a single frame should stay on the screen + double frame_time = (1000.0 / reader->info.fps.ToDouble()); - // Get the start time (to track how long a frame takes to render) - const Time t1 = Time::getCurrentTime(); + // Get the start time (to track how long a frame takes to render) + const Time t1 = Time::getCurrentTime(); - // Get the current video frame (if it's different) - frame = getFrame(); + // Get the current video frame (if it's different) + frame = getFrame(); - // Experimental Pausing Code (if frame has not changed) - if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length)) { - speed = 0; - sleep(frame_time); - continue; - } + // Experimental Pausing Code (if frame has not changed) + if ((speed == 0 && video_position == last_video_position) || (video_position > reader->info.video_length)) { + speed = 0; + sleep(frame_time); + continue; + } - // Set the video frame on the video thread and render frame - videoPlayback->frame = frame; - videoPlayback->render.signal(); - videoPlayback->rendered.wait(); + // Set the video frame on the video thread and render frame + videoPlayback->frame = frame; + videoPlayback->render.signal(); + videoPlayback->rendered.wait(); - // Keep track of the last displayed frame - last_video_position = video_position; + // Keep track of the last displayed frame + last_video_position = video_position; - // How many frames ahead or behind is the video thread? - int video_frame_diff = 0; - if (reader->info.has_audio && reader->info.has_video) { - if (speed != 1) - // Set audio frame again (since we are not in normal speed, and not paused) - audioPlayback->Seek(video_position); + // How many frames ahead or behind is the video thread? + int video_frame_diff = 0; + if (reader->info.has_audio && reader->info.has_video) { + if (speed != 1) + // Set audio frame again (since we are not in normal speed, and not paused) + audioPlayback->Seek(video_position); - // Only calculate this if a reader contains both an audio and video thread - audio_position = audioPlayback->getCurrentFramePosition(); - video_frame_diff = video_position - audio_position; - } + // Only calculate this if a reader contains both an audio and video thread + audio_position = audioPlayback->getCurrentFramePosition(); + video_frame_diff = video_position - audio_position; + } - // Get the end time (to track how long a frame takes to render) - const Time t2 = Time::getCurrentTime(); + // Get the end time (to track how long a frame takes to render) + const Time t2 = Time::getCurrentTime(); - // Determine how many milliseconds it took to render the frame - int64 render_time = t2.toMilliseconds() - t1.toMilliseconds(); + // Determine how many milliseconds it took to render the frame + int64 render_time = t2.toMilliseconds() - t1.toMilliseconds(); - // Calculate the amount of time to sleep (by subtracting the render time) - int sleep_time = int(frame_time - render_time); + // Calculate the amount of time to sleep (by subtracting the render time) + int sleep_time = int(frame_time - render_time); - // Adjust drift (if more than a few frames off between audio and video) - if (video_frame_diff > 0 && reader->info.has_audio && reader->info.has_video) - // Since the audio and video threads are running independently, they will quickly get out of sync. - // To fix this, we calculate how far ahead or behind the video frame is, and adjust the amount of time - // the frame is displayed on the screen (i.e. the sleep time). If a frame is ahead of the audio, - // we sleep for longer. If a frame is behind the audio, we sleep less (or not at all), in order for - // the video to catch up. - sleep_time += (video_frame_diff * (1000.0 / reader->info.fps.ToDouble())); + // Adjust drift (if more than a few frames off between audio and video) + if (video_frame_diff > 0 && reader->info.has_audio && reader->info.has_video) + // Since the audio and video threads are running independently, they will quickly get out of sync. + // To fix this, we calculate how far ahead or behind the video frame is, and adjust the amount of time + // the frame is displayed on the screen (i.e. the sleep time). If a frame is ahead of the audio, + // we sleep for longer. If a frame is behind the audio, we sleep less (or not at all), in order for + // the video to catch up. + sleep_time += (video_frame_diff * (1000.0 / reader->info.fps.ToDouble())); - else if (video_frame_diff < -4 && reader->info.has_audio && reader->info.has_video) { - // Skip frame(s) to catch up to the audio (if more than 4 frames behind) - video_position++; - sleep_time = 0; - } + else if (video_frame_diff < -4 && reader->info.has_audio && reader->info.has_video) { + // Skip frame(s) to catch up to the audio (if more than 4 frames behind) + video_position++; + sleep_time = 0; + } - // Sleep (leaving the video frame on the screen for the correct amount of time) - if (sleep_time > 0) sleep(sleep_time); + // Sleep (leaving the video frame on the screen for the correct amount of time) + if (sleep_time > 0) sleep(sleep_time); - // Debug output - std::cout << "frame: " << video_position << ", video frame diff: " << video_frame_diff << std::endl; - } + } } // Get the next displayed frame (based on speed and direction) diff --git a/src/QtPlayer.cpp b/src/QtPlayer.cpp index 754d1c53..8deedfa9 100644 --- a/src/QtPlayer.cpp +++ b/src/QtPlayer.cpp @@ -65,8 +65,6 @@ void QtPlayer::SetSource(const std::string &source) void QtPlayer::Play() { - cout << "PLAY() on QTPlayer" << endl; - // Set mode to playing, and speed to normal mode = PLAYBACK_PLAY; Speed(1);