From c9b74ec96d5eb034cce80583f82c845d9c139ad3 Mon Sep 17 00:00:00 2001 From: Nelson Ho Date: Fri, 6 Jan 2017 20:48:47 -0500 Subject: [PATCH] Added fixes for audio sample distribution to clean up missing or extra audio samples that were at frame boundaries --- include/FrameMapper.h | 4 ++ src/FrameMapper.cpp | 106 ++++++++++++++++++++++++++++++++++-------- src/Timeline.cpp | 5 ++ 3 files changed, 95 insertions(+), 20 deletions(-) diff --git a/include/FrameMapper.h b/include/FrameMapper.h index ecc20c85..784c5cd1 100644 --- a/include/FrameMapper.h +++ b/include/FrameMapper.h @@ -147,6 +147,7 @@ namespace openshot CacheMemory final_cache; // Cache of actual Frame objects bool is_dirty; // When this is true, the next call to GetFrame will re-init the mapping AVAudioResampleContext *avr; // Audio resampling context object + long int timeline_frame_offset; // Timeline frame offset // Internal methods used by init void AddField(long int frame); @@ -175,6 +176,9 @@ namespace openshot /// Change frame rate or audio mapping details void ChangeMapping(Fraction target_fps, PulldownType pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout); + // Set offset relative to parent timeline + void SetTimelineFrameOffset(long int offset); + /// Close the openshot::FrameMapper and internal reader void Close(); diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index 0e6c852f..fcf90eb9 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -31,7 +31,7 @@ using namespace std; using namespace openshot; FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType target_pulldown, int target_sample_rate, int target_channels, ChannelLayout target_channel_layout) : - reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL) + reader(reader), target(target), pulldown(target_pulldown), is_dirty(true), avr(NULL), timeline_frame_offset(0) { // Set the original frame rate from the reader original = Fraction(reader->info.fps.num, reader->info.fps.den); @@ -231,7 +231,7 @@ void FrameMapper::Init() if (field % 2 == 0 && field > 0) { // New frame number - long int frame_number = field / 2; + long int frame_number = field / 2 + timeline_frame_offset; // Set the bottom frame if (f.isOdd) @@ -255,7 +255,7 @@ void FrameMapper::Init() if (original_samples >= remaining_samples) { // Take all that we need, and break loop - end_samples_position += remaining_samples; + end_samples_position += remaining_samples - 1; remaining_samples = 0; } else { @@ -418,17 +418,23 @@ tr1::shared_ptr FrameMapper::GetFrame(long int requested_frame) throw(Rea // Get # of channels in the actual frame int channels_in_frame = mapped_frame->GetAudioChannelsCount(); - int samples_in_frame = Frame::GetSamplesPerFrame(frame_number, target, mapped_frame->SampleRate(), channels_in_frame); + int samples_in_frame = Frame::GetSamplesPerFrame(frame_number + timeline_frame_offset, target, mapped_frame->SampleRate(), channels_in_frame); // Determine if mapped frame is identical to source frame + // including audio sample distribution according to mapped.Samples, + // and frame_number. In some cases such as end of stream, the reader + // will return a frame with a different frame number. In these cases, + // we cannot use the frame as is, nor can we modify the frame number, + // otherwise the reader's cache object internals become invalid. if (info.sample_rate == mapped_frame->SampleRate() && info.channels == mapped_frame->GetAudioChannelsCount() && info.channel_layout == mapped_frame->ChannelsLayout() && + mapped.Samples.total == mapped_frame->GetAudioSamplesCount() && + mapped.Samples.frame_start == mapped.Odd.Frame && + mapped.Samples.sample_start == 0 && + mapped_frame->number == frame_number &&// in some conditions (e.g. end of stream) info.fps.num == reader->info.fps.num && info.fps.den == reader->info.fps.den) { - // Set frame # on mapped frame - mapped_frame->SetFrameNumber(frame_number); - // Add original frame to cache, and skip the rest (for performance reasons) final_cache.Add(mapped_frame); continue; @@ -456,13 +462,64 @@ tr1::shared_ptr FrameMapper::GetFrame(long int requested_frame) throw(Rea frame->AddImage(tr1::shared_ptr(new QImage(*even_frame->GetImage())), false); } + // Resample audio on frame (if needed) + bool need_resampling = false; + if (info.has_audio && + (info.sample_rate != frame->SampleRate() || + info.channels != frame->GetAudioChannelsCount() || + info.channel_layout != frame->ChannelsLayout())) + // Resample audio and correct # of channels if needed + need_resampling = true; + + // create a copy of mapped.Samples that will be used by copy loop + SampleRange copy_samples = mapped.Samples; + + if (need_resampling) + { + // Resampling needed, modify copy of SampleRange object that + // includes some additional input samples on first iteration, + // and continues the offset to ensure that the sample rate + // converter isn't input limited. + const int EXTRA_INPUT_SAMPLES = 20; + + // Extend end sample count by an addtional EXTRA_INPUT_SAMPLES samples + copy_samples.sample_end += EXTRA_INPUT_SAMPLES; + int samples_per_end_frame = + Frame::GetSamplesPerFrame(copy_samples.frame_end, original, + reader->info.sample_rate, reader->info.channels); + if (copy_samples.sample_end >= samples_per_end_frame) + { + // check for wrapping + copy_samples.frame_end++; + copy_samples.sample_end -= samples_per_end_frame; + } + copy_samples.total += EXTRA_INPUT_SAMPLES; + + if (avr) { + // Sample rate conversion has been allocated on this clip, so + // this is not the first iteration. Extend start position by + // EXTRA_INPUT_SAMPLES to keep step with previous frame + copy_samples.sample_start += EXTRA_INPUT_SAMPLES; + int samples_per_start_frame = + Frame::GetSamplesPerFrame(copy_samples.frame_start, original, + reader->info.sample_rate, reader->info.channels); + if (copy_samples.sample_start >= samples_per_start_frame) + { + // check for wrapping + copy_samples.frame_start++; + copy_samples.sample_start -= samples_per_start_frame; + } + copy_samples.total -= EXTRA_INPUT_SAMPLES; + } + } + // Copy the samples int samples_copied = 0; - int starting_frame = mapped.Samples.frame_start; - while (info.has_audio && samples_copied < mapped.Samples.total) + int starting_frame = copy_samples.frame_start; + while (info.has_audio && samples_copied < copy_samples.total) { // Init number of samples to copy this iteration - int remaining_samples = mapped.Samples.total - samples_copied; + int remaining_samples = copy_samples.total - samples_copied; int number_to_copy = 0; // Loop through each channel @@ -472,17 +529,17 @@ tr1::shared_ptr FrameMapper::GetFrame(long int requested_frame) throw(Rea tr1::shared_ptr original_frame = GetOrCreateFrame(starting_frame); int original_samples = original_frame->GetAudioSamplesCount(); - if (starting_frame == mapped.Samples.frame_start) + if (starting_frame == copy_samples.frame_start) { // Starting frame (take the ending samples) - number_to_copy = original_samples - mapped.Samples.sample_start; + number_to_copy = original_samples - copy_samples.sample_start; if (number_to_copy > remaining_samples) number_to_copy = remaining_samples; // Add samples to new frame - frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.Samples.sample_start, number_to_copy, 1.0); + frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + copy_samples.sample_start, number_to_copy, 1.0); } - else if (starting_frame > mapped.Samples.frame_start && starting_frame < mapped.Samples.frame_end) + else if (starting_frame > copy_samples.frame_start && starting_frame < copy_samples.frame_end) { // Middle frame (take all samples) number_to_copy = original_samples; @@ -495,7 +552,7 @@ tr1::shared_ptr FrameMapper::GetFrame(long int requested_frame) throw(Rea else { // Ending frame (take the beginning samples) - number_to_copy = mapped.Samples.sample_end; + number_to_copy = copy_samples.sample_end + 1; if (number_to_copy > remaining_samples) number_to_copy = remaining_samples; @@ -510,10 +567,7 @@ tr1::shared_ptr FrameMapper::GetFrame(long int requested_frame) throw(Rea } // Resample audio on frame (if needed) - if (info.has_audio && - (info.sample_rate != frame->SampleRate() || - info.channels != frame->GetAudioChannelsCount() || - info.channel_layout != frame->ChannelsLayout())) + if (need_resampling) // Resample audio and correct # of channels if needed ResampleMappedAudio(frame, mapped.Odd.Frame); @@ -688,6 +742,18 @@ void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldow } } +// Set offset relative to parent timeline +void FrameMapper::SetTimelineFrameOffset(long int offset) +{ + ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::SetTimelineFrameOffset", "offset", offset, "", -1, "", -1, "", -1, "", -1, "", -1); + + // Mark as dirty + is_dirty = true; + + // Used to correctly align audio sample distribution + timeline_frame_offset = offset; +} + // Resample audio and map channels (if needed) void FrameMapper::ResampleMappedAudio(tr1::shared_ptr frame, long int original_frame_number) { @@ -739,7 +805,7 @@ void FrameMapper::ResampleMappedAudio(tr1::shared_ptr frame, long int ori } // Update total samples & input frame size (due to bigger or smaller data types) - total_frame_samples = Frame::GetSamplesPerFrame(frame->number, target, info.sample_rate, info.channels); + total_frame_samples = Frame::GetSamplesPerFrame(frame->number + timeline_frame_offset, target, info.sample_rate, info.channels); ZmqLogger::Instance()->AppendDebugMethod("FrameMapper::ResampleMappedAudio (adjust # of samples)", "total_frame_samples", total_frame_samples, "info.sample_rate", info.sample_rate, "sample_rate_in_frame", sample_rate_in_frame, "info.channels", info.channels, "channels_in_frame", channels_in_frame, "original_frame_number", original_frame_number); diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 83317942..9682d5cf 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -121,6 +121,11 @@ void Timeline::apply_mapper_to_clip(Clip* clip) FrameMapper* clip_mapped_reader = (FrameMapper*) clip_reader; clip_mapped_reader->ChangeMapping(info.fps, PULLDOWN_NONE, info.sample_rate, info.channels, info.channel_layout); + // Update timeline offset + float time_diff = 0 - clip->Position() + clip->Start(); + int clip_offset = -round(time_diff * info.fps.ToFloat()); + clip_mapped_reader->SetTimelineFrameOffset(clip_offset); + // Update clip reader clip->Reader(clip_reader); }