From d3ef1fac138580de8902e328d3fac387011cdf84 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 29 Nov 2012 16:32:48 -0600 Subject: [PATCH] Refactored the audio waveform generation into the Timeline class. Added the ability to set the volume, and the ability to mix layers of audio samples together. Also, made the waveform's have a transparent background, so they can be layered on top of other layers. --- include/Clip.h | 7 +++++-- include/FFmpegWriter.h | 2 +- include/Frame.h | 5 ++++- src/Clip.cpp | 9 +++++---- src/FFmpegReader.cpp | 2 +- src/FFmpegWriter.cpp | 12 +++--------- src/Frame.cpp | 18 ++++++++++++++---- src/FrameMapper.cpp | 6 +++--- src/Main.cpp | 25 +++++++++++++++++-------- src/Timeline.cpp | 39 ++++++++++++++++++++++++++++++++------- 10 files changed, 85 insertions(+), 40 deletions(-) diff --git a/include/Clip.h b/include/Clip.h index 0794ae8d..a1862b05 100644 --- a/include/Clip.h +++ b/include/Clip.h @@ -75,6 +75,7 @@ namespace openshot { int layer; /// new_image, bool only_odd_lines); /// Add audio samples to a specific channel - void AddAudio(int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource); + void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource); + + /// Apply gain ramp (i.e. fading volume) + void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain); /// Composite a new image on top of the existing image void AddImage(tr1::shared_ptr new_image, float alpha); diff --git a/src/Clip.cpp b/src/Clip.cpp index 52049453..b7855b77 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -13,6 +13,7 @@ void Clip::init_settings() gravity = GRAVITY_CENTER; scale = SCALE_FIT; anchor = ANCHOR_CANVAS; + waveform = false; // Init scale curves scale_x = Keyframe(1.0); @@ -28,7 +29,7 @@ void Clip::init_settings() // Init time & volume time = Keyframe(0.0); - volume = Keyframe(100.0); + volume = Keyframe(1.0); // Init crop settings crop_gravity = GRAVITY_CENTER; @@ -307,7 +308,7 @@ tr1::shared_ptr Clip::get_time_mapped_frame(tr1::shared_ptr frame, start -= 1; for (int channel = 0; channel < channels; channel++) // Add new (slower) samples, to the frame object - new_frame->AddAudio(channel, 0, audio_cache->getSampleData(channel, start), number_of_samples, 1.0f); + new_frame->AddAudio(true, channel, 0, audio_cache->getSampleData(channel, start), number_of_samples, 1.0f); // Clean up if the final section if (time.GetRepeatFraction(frame_number).num == time.GetRepeatFraction(frame_number).den) @@ -406,7 +407,7 @@ tr1::shared_ptr Clip::get_time_mapped_frame(tr1::shared_ptr frame, // Add the newly resized audio samples to the current frame for (int channel = 0; channel < channels; channel++) // Add new (slower) samples, to the frame object - new_frame->AddAudio(channel, 0, buffer->getSampleData(channel), number_of_samples, 1.0f); + new_frame->AddAudio(true, channel, 0, buffer->getSampleData(channel), number_of_samples, 1.0f); // Clean up buffer = NULL; @@ -427,7 +428,7 @@ tr1::shared_ptr Clip::get_time_mapped_frame(tr1::shared_ptr frame, // Add reversed samples to the frame object for (int channel = 0; channel < channels; channel++) - new_frame->AddAudio(channel, 0, samples->getSampleData(channel), number_of_samples, 1.0f); + new_frame->AddAudio(true, channel, 0, samples->getSampleData(channel), number_of_samples, 1.0f); } diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index bec87b90..d1a847fc 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -872,7 +872,7 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int f = CreateFrame(starting_frame_number); // Add samples for current channel to the frame - f->AddAudio(channel_filter, start, iterate_channel_buffer, samples, 1.0f); + f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 1.0f); #pragma omp critical (openshot_cache) // Add or update cache diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 9af57b37..60802c42 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -146,7 +146,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, i } // Set audio export options -void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, int bit_rate, bool visualize) +void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, int bit_rate) { // Set audio options if (codec.length() > 0) @@ -172,7 +172,6 @@ void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate // Enable / Disable audio info.has_audio = has_audio; - info.visualize = visualize; } // Set custom options (some codecs accept additional params) @@ -1155,13 +1154,8 @@ void FFmpegWriter::process_video_packet(tr1::shared_ptr frame) AVFrame *frame_source = NULL; const Magick::PixelPacket *pixel_packets = NULL; - // If visualizing waveform (replace image with waveform image) - if (!info.visualize) - // Get a list of pixels from source image - pixel_packets = frame->GetPixels(); - else - // Get a list of pixels from waveform image - pixel_packets = frame->GetWaveformPixels(source_image_width, source_image_height); + // Get a list of pixels from source image + pixel_packets = frame->GetPixels(); // Init AVFrame for source image & final (converted image) frame_source = allocate_avframe(PIX_FMT_RGB24, source_image_width, source_image_height, &bytes_source); diff --git a/src/Frame.cpp b/src/Frame.cpp index cfa554c6..50087f22 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -199,7 +199,8 @@ tr1::shared_ptr Frame::GetWaveform(int width, int height) } // Create image - wave_image = tr1::shared_ptr(new Magick::Image(Magick::Geometry(total_width, total_height), Magick::Color("#000000"))); + wave_image = tr1::shared_ptr(new Magick::Image(Magick::Geometry(total_width, total_height), Magick::Color("none"))); + wave_image->backgroundColor(Magick::Color("none")); // Draw the waveform wave_image->draw(lines); @@ -216,7 +217,8 @@ tr1::shared_ptr Frame::GetWaveform(int width, int height) else { // No audio samples present - wave_image = tr1::shared_ptr(new Magick::Image(Magick::Geometry(width, height), Magick::Color("#000000"))); + wave_image = tr1::shared_ptr(new Magick::Image(Magick::Geometry(width, height), Magick::Color("none"))); + wave_image->backgroundColor(Magick::Color("none")); // Add Channel Label lines.push_back(Magick::DrawableStrokeColor("#ffffff")); @@ -524,19 +526,27 @@ void Frame::AddImage(tr1::shared_ptr new_image, float alpha) } // Add audio samples to a specific channel -void Frame::AddAudio(int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) +void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) { // Extend audio buffer (if needed) if (destStartSample + numSamples > audio->getNumSamples()) audio->setSize(audio->getNumChannels(), destStartSample + numSamples, true, true, false); // Always clear the range of samples first - audio->clear(destChannel, destStartSample, numSamples); + if (replaceSamples) + audio->clear(destChannel, destStartSample, numSamples); // Add samples to frame's audio buffer audio->addFrom(destChannel, destStartSample, source, numSamples, gainToApplyToSource); } +// Apply gain ramp (i.e. fading volume) +void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f) +{ + // Apply gain ramp + audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain); +} + // Experimental method to add effects to this frame void Frame::AddEffect(string name) { diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index 2f25d9c4..df7838c5 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -288,7 +288,7 @@ tr1::shared_ptr FrameMapper::GetFrame(int requested_frame) throw(ReaderCl number_to_copy = remaining_samples; // Add samples to new frame - frame->AddAudio(channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.Samples.sample_start, number_to_copy, 1.0); + frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.Samples.sample_start, number_to_copy, 1.0); } else if (starting_frame > mapped.Samples.frame_start && starting_frame < mapped.Samples.frame_end) { @@ -298,7 +298,7 @@ tr1::shared_ptr FrameMapper::GetFrame(int requested_frame) throw(ReaderCl number_to_copy = remaining_samples; // Add samples to new frame - frame->AddAudio(channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0); + frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0); } else { @@ -308,7 +308,7 @@ tr1::shared_ptr FrameMapper::GetFrame(int requested_frame) throw(ReaderCl number_to_copy = remaining_samples; // Add samples to new frame - frame->AddAudio(channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0); + frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0); } } diff --git a/src/Main.cpp b/src/Main.cpp index 684ac854..0b47bc1f 100644 --- a/src/Main.cpp +++ b/src/Main.cpp @@ -60,16 +60,24 @@ int main() //Clip c1(new FFmpegReader("/home/jonathan/Apps/videcho_site/media/user_files/videos/bd0bf442-3221-11e2-8bf6-001fd00ee3aa.webm")); Clip c1(new FFmpegReader("/home/jonathan/Music/lonely island/B004YRCAIU_(disc_1)_02_-_Mama_[Explicit].mp3")); //Clip c1(new FFmpegReader("/home/jonathan/sintel.webm")); - Clip c2(new ImageReader("/home/jonathan/Desktop/logo1.png")); - Clip c3(new ImageReader("/home/jonathan/Desktop/logo1.png")); + Clip c2(new ImageReader("/home/jonathan/Desktop/openshot_style1.png")); + Clip c3(new FFmpegReader("/home/jonathan/Music/lonely island/B004YRCAUI_(disc_1)_16_-_Japan.mp3")); //Clip c3(new FFmpegReader("/home/jonathan/Desktop/IncognitoCory_-_April_Song.mp3")); c1.Position(0.0); c1.gravity = GRAVITY_CENTER; c1.scale = SCALE_FIT; c1.End(20); + c1.Layer(2); + c1.volume.AddPoint(1, 0.0); + c1.volume.AddPoint(100, 0.5); + c1.volume.AddPoint(200, 0.0); + c1.volume.AddPoint(300, 0.5); + c1.Waveform(true); + //c1.scale_x.AddPoint(1, 0.5, LINEAR); + //c1.scale_y.AddPoint(1, 0.5, LINEAR); c2.Position(0.0); - c2.Layer(2); + c2.Layer(1); c2.gravity = GRAVITY_BOTTOM_RIGHT; c2.scale = SCALE_NONE; c2.alpha.AddPoint(1, 1); @@ -79,9 +87,10 @@ int main() c2.location_y.AddPoint(1, -0.02); c2.End(20); - c3.Layer(3); + c3.Layer(0); c3.End(20); c3.gravity = GRAVITY_CENTER; + c3.volume.AddPoint(1, 0.5); //c3.scale_x.AddPoint(1, 0.1); //c3.scale_x.AddPoint(300, 2.0); //c3.scale_y.AddPoint(1, 0.1); @@ -162,8 +171,8 @@ int main() // Add clips t.AddClip(&c1); - //t.AddClip(&c2); - //t.AddClip(&c3); + t.AddClip(&c2); + t.AddClip(&c3); // Create a writer @@ -172,7 +181,7 @@ int main() // Set options //w.SetAudioOptions(true, "libmp3lame", 44100, 2, 128000, false); - w.SetAudioOptions(true, "libvorbis", 44100, 2, 128000, true); + w.SetAudioOptions(true, "libvorbis", 44100, 2, 128000); w.SetVideoOptions(true, "libvpx", Fraction(24, 1), 624, 348, Fraction(1,1), false, false, 2000000); // Prepare Streams @@ -184,7 +193,7 @@ int main() // Output stream info w.OutputStreamInfo(); - for (int frame = 1; frame <= 600; frame++) + for (int frame = 1; frame <= 300; frame++) { tr1::shared_ptr f = t.GetFrame(frame); if (f) diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 0d986a0a..94a6551f 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -56,19 +56,44 @@ void Timeline::add_layer(tr1::shared_ptr new_frame, Clip* source_clip, in tr1::shared_ptr source_frame; #pragma omp critical (reader_lock) source_frame = tr1::shared_ptr(source_clip->GetFrame(clip_frame_number)); - tr1::shared_ptr source_image = source_frame->GetImage(); - - // Get some basic image properties - int source_width = source_image->columns(); - int source_height = source_image->rows(); + tr1::shared_ptr source_image; /* CREATE BACKGROUND COLOR - needed if this is the 1st layer */ if (new_frame->GetImage()->columns() == 1) new_frame->AddColor(width, height, "#000000"); - /* COPY AUDIO */ + /* COPY AUDIO - with correct volume */ for (int channel = 0; channel < source_frame->GetAudioChannelsCount(); channel++) - new_frame->AddAudio(channel, 0, source_frame->GetAudioSamples(channel), source_frame->GetAudioSamplesCount(), 1.0f); + { + float initial_volume = 1.0f; + float previous_volume = source_clip->volume.GetValue(clip_frame_number - 1); // previous frame's percentage of volume (0 to 1) + float volume = source_clip->volume.GetValue(clip_frame_number); // percentage of volume (0 to 1) + + // If no ramp needed, set initial volume = clip's volume + if (isEqual(previous_volume, volume)) + initial_volume = volume; + + // Apply ramp to source frame (if needed) + if (!isEqual(previous_volume, volume)) + source_frame->ApplyGainRamp(channel, 0, source_frame->GetAudioSamplesCount(), previous_volume, volume); + + // Copy audio samples (and set initial volume). Mix samples with existing audio samples. The gains are added together, to + // be sure to set the gain's correctly, so the sum does not exceed 1.0 (of audio distortion will happen). + new_frame->AddAudio(false, channel, 0, source_frame->GetAudioSamples(channel), source_frame->GetAudioSamplesCount(), initial_volume); + + } + + /* GET IMAGE DATA - OR GENERATE IT */ + if (!source_clip->Waveform()) + // Get actual frame image data + source_image = source_frame->GetImage(); + else + // Generate Waveform Dynamically (the size of the timeline) + source_image = source_frame->GetWaveform(width, height); + + // Get some basic image properties + int source_width = source_image->columns(); + int source_height = source_image->rows(); /* ALPHA & OPACITY */ if (source_clip->alpha.GetValue(clip_frame_number) != 0)