openshot/libopenshot
0
0
Fork 0
mirror of https://github.com/OpenShot/libopenshot.git synced 2026-03-02 08:53:52 -08:00
Code Issues Packages Projects Releases Wiki Activity

Large fix for FrameMapper when resampling audio - to correctly apply the EXTRA_INPUT_SAMPLES, to prevent the resampler from becoming input limited.

Browse Source 
- Fixed some unit tests
- Added new convenience methods for SampleRange, to extend either side, or shift it left or right
- Added new SampleRange unit tests
- No more audio pops at the beginning of clips which are being resampled!!!
- Time remapping now works perfectly smoothly, no more pops or crashes!!!
This commit is contained in:
Jonathan Thomas
2023-03-11 16:50:28 -06:00
parent ec1bd0185a
commit 45c273bf42
4 changed files with 301 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/FrameMapper.cpp
View File

@@ -549,40 +549,18 @@ std::shared_ptr<Frame> FrameMapper::GetFrame(int64_t requested_frame)
}
}
// Resampling needed, modify copy of SampleRange object that
// includes some additional input samples on first iteration,
// and continues the offset to ensure that the sample rate
// converter isn't input limited.
const int EXTRA_INPUT_SAMPLES = 100;
// Resampling needed, modify copy of SampleRange object that includes some additional input samples on
// first iteration, and continues the offset to ensure that the resampler is not input limited.
const int EXTRA_INPUT_SAMPLES = 48;
// Extend end sample count by an additional EXTRA_INPUT_SAMPLES samples
copy_samples.sample_end += EXTRA_INPUT_SAMPLES;
int samples_per_end_frame =
Frame::GetSamplesPerFrame(copy_samples.frame_end, original,
reader->info.sample_rate, reader->info.channels);
if (copy_samples.sample_end >= samples_per_end_frame)
{
// check for wrapping
copy_samples.frame_end++;
copy_samples.sample_end -= samples_per_end_frame;
}
copy_samples.total += EXTRA_INPUT_SAMPLES;
if (avr) {
// Sample rate conversion has been allocated on this clip, so
// this is not the first iteration. Extend start position by
// EXTRA_INPUT_SAMPLES to keep step with previous frame
copy_samples.sample_start += EXTRA_INPUT_SAMPLES;
int samples_per_start_frame =
Frame::GetSamplesPerFrame(copy_samples.frame_start, original,
reader->info.sample_rate, reader->info.channels);
if (copy_samples.sample_start >= samples_per_start_frame)
{
// check for wrapping
copy_samples.frame_start++;
copy_samples.sample_start -= samples_per_start_frame;
}
copy_samples.total -= EXTRA_INPUT_SAMPLES;
if (!avr) {
// This is the first iteration, and we need to extend # of samples for this frame
// Extend sample count range by an additional EXTRA_INPUT_SAMPLES
copy_samples.Extend(EXTRA_INPUT_SAMPLES, original, reader->info.sample_rate, reader->info.channels, is_increasing);
} else {
// Sample rate conversion has already been allocated on this clip, so
// this is not the first iteration. Shift position by EXTRA_INPUT_SAMPLES to correctly align samples
copy_samples.Shift(EXTRA_INPUT_SAMPLES, original, reader->info.sample_rate, reader->info.channels, is_increasing);
}
}

82
src/FrameMapper.h
View File

@@ -80,6 +80,86 @@ namespace openshot
int64_t frame_end;
int sample_end;
/// Extend SampleRange on either side
void Extend(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side) {
int remaining_samples = samples;
while (remaining_samples > 0) {
if (right_side) {
// Extend range to the right
int samples_per_frame = Frame::GetSamplesPerFrame(frame_end, fps, sample_rate, channels);
if (remaining_samples + sample_end < samples_per_frame) {
sample_end += remaining_samples;
remaining_samples = 0;
} else {
frame_end++;
remaining_samples -= (samples_per_frame - sample_end);
sample_end = 0;
}
} else {
// Extend range to the left
if (sample_start - remaining_samples >= 0) {
sample_start -= remaining_samples;
remaining_samples = 0;
} else {
frame_start--;
remaining_samples -= (sample_start + 1);
sample_start = Frame::GetSamplesPerFrame(frame_start, fps, sample_rate, channels) - 1;
}
}
}
// Increase total
total += samples;
}
/// Shrink SampleRange on either side
void Shrink(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side) {
int remaining_samples = samples;
while (remaining_samples > 0) {
if (right_side) {
// Shrink range on the right
if (sample_end - remaining_samples >= 0) {
sample_end -= remaining_samples;
remaining_samples = 0;
} else {
frame_end--;
int samples_per_frame = Frame::GetSamplesPerFrame(frame_end, fps, sample_rate, channels);
remaining_samples -= (sample_end + 1);
sample_end = samples_per_frame - 1;
}
} else {
// Shrink range on the left
int samples_per_frame = Frame::GetSamplesPerFrame(frame_start, fps, sample_rate, channels);
if (sample_start + remaining_samples < samples_per_frame) {
sample_start += remaining_samples;
remaining_samples = 0;
} else {
frame_start++;
remaining_samples -= (samples_per_frame - sample_start);
sample_start = 0;
}
}
}
// Reduce total
total -= samples;
}
void Shift(int64_t samples, openshot::Fraction fps, int sample_rate, int channels, bool right_side) {
// Extend each side of the range (to SHIFT the range) by adding (or subtracting) from both sides
// For example: [ range ]
// For example: [ range ]
if (right_side) {
// SHIFT both sides to the right
Extend(samples, fps, sample_rate, channels, true);
Shrink(samples, fps, sample_rate, channels, false);
} else {
// SHIFT both sides to the left
Extend(samples, fps, sample_rate, channels, false);
Shrink(samples, fps, sample_rate, channels, true);
}
}
int total;
};
@@ -128,7 +208,7 @@ namespace openshot
CacheMemory final_cache; // Cache of actual Frame objects
bool is_dirty; // When this is true, the next call to GetFrame will re-init the mapping
float parent_position; // Position of parent clip (which is used to generate the audio mapping)
float parent_start; // Start of parent clip (which is used to generate the audio mapping)
float parent_start; // Start of parent clip (which is used to generate the audio mapping)
int64_t previous_frame; // Used during resampling, to determine when a large gap is detected
SWRCONTEXT *avr; // Audio resampling context object

26
tests/Clip.cpp
View File

@@ -387,17 +387,9 @@ TEST_CASE( "time remapping", "[libopenshot][clip]" )
clip.Start(0.0);
// Set time keyframe (4X speed REVERSE)
//clip.time.AddPoint(1, original_video_length, openshot::LINEAR);
//clip.time.AddPoint(original_video_length, 1.0, openshot::LINEAR);
// Set time keyframe (4X speed FORWARD)
//clip.time.AddPoint(1, 1.0, openshot::LINEAR);
//clip.time.AddPoint(original_video_length / 2, original_video_length, openshot::LINEAR);
// Set time keyframe (1/4X speed FORWARD)
//clip.time.AddPoint(1, 1.0, openshot::LINEAR);
//clip.time.AddPoint(original_video_length * 4, original_video_length, openshot::LINEAR);
clip.time.AddPoint(1, original_video_length, openshot::LINEAR);
clip.time.AddPoint(original_video_length, 1.0, openshot::LINEAR);
// TODO: clip.Duration() != clip.Reader->info.duration
// Set clip length based on time-values
if (clip.time.GetLength() > 1) {
@@ -424,9 +416,7 @@ TEST_CASE( "time remapping", "[libopenshot][clip]" )
t1.info.channels);
std::shared_ptr<Frame> f = t1.GetFrame(frame);
if (expected_sample_count != f->GetAudioSamplesCount()) {
CHECK(expected_sample_count == f->GetAudioSamplesCount());
}
CHECK(expected_sample_count == f->GetAudioSamplesCount());
}
// Clear cache
@@ -440,9 +430,7 @@ TEST_CASE( "time remapping", "[libopenshot][clip]" )
t1.info.channels);
std::shared_ptr<Frame> f = t1.GetFrame(frame);
if (expected_sample_count != f->GetAudioSamplesCount()) {
CHECK(expected_sample_count == f->GetAudioSamplesCount());
}
CHECK(expected_sample_count == f->GetAudioSamplesCount());
}
t1.Close();
@@ -480,9 +468,7 @@ TEST_CASE( "resample_audio_8000_to_48000_reverse", "[libopenshot][clip]" )
map.info.channels);
std::shared_ptr<Frame> f = clip.GetFrame(frame);
if (expected_sample_count != f->GetAudioSamplesCount()) {
CHECK(expected_sample_count == f->GetAudioSamplesCount());
}
CHECK(expected_sample_count == f->GetAudioSamplesCount());
}
// Clear clip cache

308
tests/FrameMapper.cpp
View File

@@ -472,136 +472,136 @@ TEST_CASE( "redistribute_samples_per_frame", "[libopenshot][framemapper]" ) {
}
TEST_CASE( "Distribute samples", "[libopenshot][framemapper]" ) {
// This test verifies that audio data can be redistributed correctly
// between common and uncommon frame rates
int sample_rate = 48000;
int channels = 2;
int num_seconds = 1;
// This test verifies that audio data can be redistributed correctly
// between common and uncommon frame rates
int sample_rate = 48000;
int channels = 2;
int num_seconds = 1;
// Source frame rates (varies the # of samples per frame)
std::vector<openshot::Fraction> rates = {
openshot::Fraction(30,1),
openshot::Fraction(24,1) ,
openshot::Fraction(119,4),
openshot::Fraction(30000,1001)
};
// Source frame rates (varies the # of samples per frame)
std::vector<openshot::Fraction> rates = {
openshot::Fraction(30,1),
openshot::Fraction(24,1) ,
openshot::Fraction(119,4),
openshot::Fraction(30000,1001)
};
for (auto& frame_rate : rates) {
// Init sin wave variables
const int OFFSET = 0;
const float AMPLITUDE = 0.75;
const int NUM_SAMPLES = 100;
double angle = 0.0;
for (auto& frame_rate : rates) {
// Init sin wave variables
const int OFFSET = 0;
const float AMPLITUDE = 0.75;
const int NUM_SAMPLES = 100;
double angle = 0.0;
// Create cache object to hold test frames
openshot::CacheMemory cache;
// Create cache object to hold test frames
openshot::CacheMemory cache;
// Let's create some test frames
for (int64_t frame_number = 1; frame_number <= (frame_rate.ToFloat() * num_seconds * 2); ++frame_number) {
// Create blank frame (with specific frame #, samples, and channels)
int sample_count = openshot::Frame::GetSamplesPerFrame(frame_number, frame_rate, sample_rate, channels);
auto f = std::make_shared<openshot::Frame>(frame_number, sample_count, channels);
f->SampleRate(sample_rate);
// Let's create some test frames
for (int64_t frame_number = 1; frame_number <= (frame_rate.ToFloat() * num_seconds * 2); ++frame_number) {
// Create blank frame (with specific frame #, samples, and channels)
int sample_count = openshot::Frame::GetSamplesPerFrame(frame_number, frame_rate, sample_rate, channels);
auto f = std::make_shared<openshot::Frame>(frame_number, sample_count, channels);
f->SampleRate(sample_rate);
// Create test samples with sin wave (predictable values)
float *audio_buffer = new float[sample_count * 2];
for (int sample_number = 0; sample_number < sample_count; sample_number++) {
// Calculate sin wave
float sample_value = float(AMPLITUDE * sin(angle) + OFFSET);
audio_buffer[sample_number] = abs(sample_value);
angle += (2 * M_PI) / NUM_SAMPLES;
}
// Create test samples with sin wave (predictable values)
float *audio_buffer = new float[sample_count * 2];
for (int sample_number = 0; sample_number < sample_count; sample_number++) {
// Calculate sin wave
float sample_value = float(AMPLITUDE * sin(angle) + OFFSET);
audio_buffer[sample_number] = abs(sample_value);
angle += (2 * M_PI) / NUM_SAMPLES;
}
// Add custom audio samples to Frame (bool replaceSamples, int destChannel, int destStartSample, const float* source,
f->AddAudio(true, 0, 0, audio_buffer, sample_count, 1.0); // add channel 1
f->AddAudio(true, 1, 0, audio_buffer, sample_count, 1.0); // add channel 2
// Add custom audio samples to Frame (bool replaceSamples, int destChannel, int destStartSample, const float* source,
f->AddAudio(true, 0, 0, audio_buffer, sample_count, 1.0); // add channel 1
f->AddAudio(true, 1, 0, audio_buffer, sample_count, 1.0); // add channel 2
// Add test frame to dummy reader
cache.Add(f);
// Add test frame to dummy reader
cache.Add(f);
delete[] audio_buffer;
}
delete[] audio_buffer;
}
openshot::DummyReader r(frame_rate, 1920, 1080, sample_rate, channels, 30.0, &cache);
r.Open();
openshot::DummyReader r(frame_rate, 1920, 1080, sample_rate, channels, 30.0, &cache);
r.Open();
// Target frame rates
// Target frame rates
std::vector<openshot::Fraction> mapped_rates = {
openshot::Fraction(30,1),
openshot::Fraction(24,1),
openshot::Fraction(119,4),
openshot::Fraction(30000,1001)
};
for (auto &mapped_rate : mapped_rates) {
// Reset SIN wave
angle = 0.0;
openshot::Fraction(30,1),
openshot::Fraction(24,1),
openshot::Fraction(119,4),
openshot::Fraction(30000,1001)
};
for (auto &mapped_rate : mapped_rates) {
// Reset SIN wave
angle = 0.0;
// Map to different fps
FrameMapper map(&r, mapped_rate, PULLDOWN_NONE, sample_rate, channels, LAYOUT_STEREO);
map.info.has_audio = true;
map.Open();
// Map to different fps
FrameMapper map(&r, mapped_rate, PULLDOWN_NONE, sample_rate, channels, LAYOUT_STEREO);
map.info.has_audio = true;
map.Open();
// Loop through samples, and verify FrameMapper didn't mess up individual sample values
int num_samples = 0;
for (int frame_index = 1; frame_index <= (map.info.fps.ToInt() * num_seconds); frame_index++) {
int sample_count = map.GetFrame(frame_index)->GetAudioSamplesCount();
for (int sample_index = 0; sample_index < sample_count; sample_index++) {
// Loop through samples, and verify FrameMapper didn't mess up individual sample values
int num_samples = 0;
for (int frame_index = 1; frame_index <= (map.info.fps.ToInt() * num_seconds); frame_index++) {
int sample_count = map.GetFrame(frame_index)->GetAudioSamplesCount();
for (int sample_index = 0; sample_index < sample_count; sample_index++) {
// Calculate sin wave
float predicted_value = abs(float(AMPLITUDE * sin(angle) + OFFSET));
angle += (2 * M_PI) / NUM_SAMPLES;
// Calculate sin wave
float predicted_value = abs(float(AMPLITUDE * sin(angle) + OFFSET));
angle += (2 * M_PI) / NUM_SAMPLES;
// Verify each mapped sample value is correct (after being redistributed by the FrameMapper)
float mapped_value = map.GetFrame(frame_index)->GetAudioSample(0, sample_index, 1.0);
CHECK(predicted_value == Approx(mapped_value).margin(0.001));
}
// Increment sample value
num_samples += map.GetFrame(frame_index)->GetAudioSamplesCount();
}
// Verify each mapped sample value is correct (after being redistributed by the FrameMapper)
float mapped_value = map.GetFrame(frame_index)->GetAudioSample(0, sample_index, 1.0);
CHECK(predicted_value == Approx(mapped_value).margin(0.001));
}
// Increment sample value
num_samples += map.GetFrame(frame_index)->GetAudioSamplesCount();
}
float clip_position = 3.77;
int starting_clip_frame = round(clip_position * map.info.fps.ToFloat()) + 1;
float clip_position = 3.77;
int starting_clip_frame = round(clip_position * map.info.fps.ToFloat()) + 1;
// Create Timeline (same specs as reader)
Timeline t1(map.info.width, map.info.height, map.info.fps, map.info.sample_rate, map.info.channels,
map.info.channel_layout);
// Create Timeline (same specs as reader)
Timeline t1(map.info.width, map.info.height, map.info.fps, map.info.sample_rate, map.info.channels,
map.info.channel_layout);
Clip c1;
c1.Reader(&map);
c1.Layer(1);
c1.Position(clip_position);
c1.Start(0.0);
c1.End(10.0);
Clip c1;
c1.Reader(&map);
c1.Layer(1);
c1.Position(clip_position);
c1.Start(0.0);
c1.End(10.0);
// Add clips
t1.AddClip(&c1);
t1.Open();
// Add clips
t1.AddClip(&c1);
t1.Open();
// Reset SIN wave
angle = 0.0;
// Reset SIN wave
angle = 0.0;
for (int frame_index = starting_clip_frame; frame_index < (starting_clip_frame + (t1.info.fps.ToFloat() * num_seconds)); frame_index++) {
for (int sample_index = 0; sample_index < t1.GetFrame(frame_index)->GetAudioSamplesCount(); sample_index++) {
// Calculate sin wave
float predicted_value = abs(float(AMPLITUDE * sin(angle) + OFFSET));
angle += (2 * M_PI) / NUM_SAMPLES;
for (int frame_index = starting_clip_frame; frame_index < (starting_clip_frame + (t1.info.fps.ToFloat() * num_seconds)); frame_index++) {
for (int sample_index = 0; sample_index < t1.GetFrame(frame_index)->GetAudioSamplesCount(); sample_index++) {
// Calculate sin wave
float predicted_value = abs(float(AMPLITUDE * sin(angle) + OFFSET));
angle += (2 * M_PI) / NUM_SAMPLES;
// Verify each mapped sample value is correct (after being redistributed by the FrameMapper)
float timeline_value = t1.GetFrame(frame_index)->GetAudioSample(0, sample_index, 1.0);
CHECK(predicted_value == Approx(timeline_value).margin(0.001));
}
}
// Verify each mapped sample value is correct (after being redistributed by the FrameMapper)
float timeline_value = t1.GetFrame(frame_index)->GetAudioSample(0, sample_index, 1.0);
CHECK(predicted_value == Approx(timeline_value).margin(0.001));
}
}
// Close mapper
map.Close();
t1.Close();
}
// Close mapper
map.Close();
t1.Close();
}
// Clean up reader
r.Close();
cache.Clear();
// Clean up reader
r.Close();
cache.Clear();
} // for rates
} // for rates
}
TEST_CASE( "PrintMapping", "[libopenshot][framemapper]" )
@@ -651,3 +651,101 @@ TEST_CASE( "Json", "[libopenshot][framemapper]" )
CHECK(map.info.sample_rate == 48000);
CHECK(map.info.fps.num == 30);
}
TEST_CASE( "SampleRange", "[libopenshot][framemapper]")
{
openshot::Fraction fps(30, 1);
int sample_rate = 44100;
int channels = 2;
int64_t start_frame = 10;
int start_sample = 0;
int total_samples = Frame::GetSamplesPerFrame(start_frame, fps, sample_rate, channels);
int64_t end_frame = 10;
int end_sample = (total_samples - 1);
SampleRange Samples = {start_frame, start_sample, end_frame, end_sample, total_samples };
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 0);
CHECK(Samples.frame_end == 10);
CHECK(Samples.sample_end == 1469);
// ------ RIGHT -------
// Extend range to the RIGHT
Samples.Extend(50, fps, sample_rate, channels, true);
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 0);
CHECK(Samples.frame_end == 11);
CHECK(Samples.sample_end == 49);
CHECK(Samples.total == total_samples + 50);
// Shrink range from the RIGHT
Samples.Shrink(50, fps, sample_rate, channels, true);
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 0);
CHECK(Samples.frame_end == 10);
CHECK(Samples.sample_end == 1469);
CHECK(Samples.total == total_samples);
// ------ LEFT -------
// Extend range to the LEFT
Samples.Extend(50, fps, sample_rate, channels, false);
CHECK(Samples.frame_start == 9);
CHECK(Samples.sample_start == 1420);
CHECK(Samples.frame_end == 10);
CHECK(Samples.sample_end == 1469);
CHECK(Samples.total == total_samples + 50);
// Shrink range from the LEFT
Samples.Shrink(50, fps, sample_rate, channels, false);
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 0);
CHECK(Samples.frame_end == 10);
CHECK(Samples.sample_end == 1469);
CHECK(Samples.total == total_samples);
// ------ SHIFT -------
// Shift range to the RIGHT
Samples.Shift(50, fps, sample_rate, channels, true);
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 50);
CHECK(Samples.frame_end == 11);
CHECK(Samples.sample_end == 49);
CHECK(Samples.total == total_samples);
// Shift range to the LEFT
Samples.Shift(50, fps, sample_rate, channels, false);
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 0);
CHECK(Samples.frame_end == 10);
CHECK(Samples.sample_end == 1469);
CHECK(Samples.total == total_samples);
// Shift range to the RIGHT
Samples.Shift(50, fps, sample_rate, channels, true);
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 50);
CHECK(Samples.frame_end == 11);
CHECK(Samples.sample_end == 49);
CHECK(Samples.total == total_samples);
// Shift range to the LEFT
Samples.Shift(75, fps, sample_rate, channels, false);
CHECK(Samples.frame_start == 9);
CHECK(Samples.sample_start == 1445);
CHECK(Samples.frame_end == 10);
CHECK(Samples.sample_end == 1444);
CHECK(Samples.total == total_samples);
// Shift range to the RIGHT
Samples.Shift(25, fps, sample_rate, channels, true);
CHECK(Samples.frame_start == 10);
CHECK(Samples.sample_start == 0);
CHECK(Samples.frame_end == 10);
CHECK(Samples.sample_end == 1469);
CHECK(Samples.total == total_samples);
}