Refactored the audio waveform generation into the Timeline class. Added the ability to set the volume, and the ability to mix layers of audio samples together. Also, made the waveform's have a transparent background, so they can be layered on top of other layers.

2026-03-02 08:53:52 -08:00 · 2012-11-29 16:32:48 -06:00
parent afd9b1d6d3
commit d3ef1fac13
10 changed files with 85 additions and 40 deletions
--- a/include/Clip.h
+++ b/include/Clip.h
@@ -75,6 +75,7 @@ namespace openshot {
 		int layer; ///<The layer this clip is on. Lower clips are covered up by higher clips.
 		float start; ///<The position in seconds to start playing (used to trim the beginning of a clip)
 		float end; ///<The position in seconds to end playing (used to trim the ending of a clip)
+		bool waveform; ///<Should a waveform be used instead of the clip's image

 		// Audio resampler (if time mapping)
 		AudioResampler *resampler;
@@ -144,13 +145,15 @@ namespace openshot {
 		int Layer() { return layer; } ///<Get layer of clip on timeline (lower number is covered by higher numbers)
 		float Start() { return start; } ///<Get start position of clip (trim start of video)
 		float End(); ///<Get end position of clip (trim end of video), which can be affected by the time curve.
-		float Duration() { End() - Start(); } ///<Get the length of this clip (in seconds)
+		float Duration() { return End() - Start(); } ///<Get the length of this clip (in seconds)
+		bool Waveform() { return waveform; } ///<Get the waveform property of this clip

 		/// Set basic properties
 		void Position(float value) { position = value; } ///<Get position on timeline
 		void Layer(int value) { layer = value; } ///<Get layer of clip on timeline (lower number is covered by higher numbers)
 		void Start(float value) { start = value; } ///<Get start position of clip (trim start of video)
 		void End(float value) { end = value; } ///<Get end position of clip (trim end of video)
+		void Waveform(bool value) { waveform = value; } ///<Set the waveform property of this clip

 		// Scale and Location curves
 		Keyframe scale_x; ///<Curve representing the horizontal scaling in percent (0 to 100)
@@ -164,7 +167,7 @@ namespace openshot {

 		// Time and Volume curves
 		Keyframe time; ///<Curve representing the frames over time to play (used for speed and direction of video)
-		Keyframe volume; ///<Curve representing the volume (0 to 100)
+		Keyframe volume; ///<Curve representing the volume (0 to 1)

 		// Crop settings and curves
 		GravityType crop_gravity; ///<Cropping needs to have a gravity to determine what side we are cropping
--- a/include/FFmpegWriter.h
+++ b/include/FFmpegWriter.h
@@ -167,7 +167,7 @@ namespace openshot
 		void RemoveScalers();

 		/// Set audio export options
-		void SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, int bit_rate, bool visualize);
+		void SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, int bit_rate);

 		/// Set the cache size (number of frames to queue before writing)
 		int SetCacheSize(int new_size) { cache_size = new_size; };
--- a/include/Frame.h
+++ b/include/Frame.h
@@ -81,7 +81,10 @@ namespace openshot
 		void AddImage(tr1::shared_ptr<Magick::Image> new_image, bool only_odd_lines);

 		/// Add audio samples to a specific channel
-		void AddAudio(int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource);
+		void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource);
+
+		/// Apply gain ramp (i.e. fading volume)
+		void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain);

 		/// Composite a new image on top of the existing image
 		void AddImage(tr1::shared_ptr<Magick::Image> new_image, float alpha);
--- a/src/Clip.cpp
+++ b/src/Clip.cpp
@@ -13,6 +13,7 @@ void Clip::init_settings()
 	gravity = GRAVITY_CENTER;
 	scale = SCALE_FIT;
 	anchor = ANCHOR_CANVAS;
+	waveform = false;

 	// Init scale curves
 	scale_x = Keyframe(1.0);
@@ -28,7 +29,7 @@ void Clip::init_settings()

 	// Init time & volume
 	time = Keyframe(0.0);
-	volume = Keyframe(100.0);
+	volume = Keyframe(1.0);

 	// Init crop settings
 	crop_gravity = GRAVITY_CENTER;
@@ -307,7 +308,7 @@ tr1::shared_ptr<Frame> Clip::get_time_mapped_frame(tr1::shared_ptr<Frame> frame,
 				start -= 1;
 			for (int channel = 0; channel < channels; channel++)
 				// Add new (slower) samples, to the frame object
-				new_frame->AddAudio(channel, 0, audio_cache->getSampleData(channel, start), number_of_samples, 1.0f);
+				new_frame->AddAudio(true, channel, 0, audio_cache->getSampleData(channel, start), number_of_samples, 1.0f);

 			// Clean up if the final section
 			if (time.GetRepeatFraction(frame_number).num == time.GetRepeatFraction(frame_number).den)
@@ -406,7 +407,7 @@ tr1::shared_ptr<Frame> Clip::get_time_mapped_frame(tr1::shared_ptr<Frame> frame,
 			// Add the newly resized audio samples to the current frame
 			for (int channel = 0; channel < channels; channel++)
 				// Add new (slower) samples, to the frame object
-				new_frame->AddAudio(channel, 0, buffer->getSampleData(channel), number_of_samples, 1.0f);
+				new_frame->AddAudio(true, channel, 0, buffer->getSampleData(channel), number_of_samples, 1.0f);

 			// Clean up
 			buffer = NULL;
@@ -427,7 +428,7 @@ tr1::shared_ptr<Frame> Clip::get_time_mapped_frame(tr1::shared_ptr<Frame> frame,

 			// Add reversed samples to the frame object
 			for (int channel = 0; channel < channels; channel++)
-				new_frame->AddAudio(channel, 0, samples->getSampleData(channel), number_of_samples, 1.0f);
+				new_frame->AddAudio(true, channel, 0, samples->getSampleData(channel), number_of_samples, 1.0f);


 		}
--- a/src/FFmpegReader.cpp
+++ b/src/FFmpegReader.cpp
@@ -872,7 +872,7 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int
 					f = CreateFrame(starting_frame_number);

 				// Add samples for current channel to the frame
-				f->AddAudio(channel_filter, start, iterate_channel_buffer, samples, 1.0f);
+				f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 1.0f);

 				#pragma omp critical (openshot_cache)
 					// Add or update cache
--- a/src/FFmpegWriter.cpp
+++ b/src/FFmpegWriter.cpp
@@ -146,7 +146,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, string codec, Fraction fps, i
 }

 // Set audio export options
-void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, int bit_rate, bool visualize)
+void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate, int channels, int bit_rate)
 {
 	// Set audio options
 	if (codec.length() > 0)
@@ -172,7 +172,6 @@ void FFmpegWriter::SetAudioOptions(bool has_audio, string codec, int sample_rate

 	// Enable / Disable audio
 	info.has_audio = has_audio;
-	info.visualize = visualize;
 }

 // Set custom options (some codecs accept additional params)
@@ -1155,13 +1154,8 @@ void FFmpegWriter::process_video_packet(tr1::shared_ptr<Frame> frame)
 		AVFrame *frame_source = NULL;
 		const Magick::PixelPacket *pixel_packets = NULL;

-		// If visualizing waveform (replace image with waveform image)
-		if (!info.visualize)
-			// Get a list of pixels from source image
-			pixel_packets = frame->GetPixels();
-		else
-			// Get a list of pixels from waveform image
-			pixel_packets = frame->GetWaveformPixels(source_image_width, source_image_height);
+		// Get a list of pixels from source image
+		pixel_packets = frame->GetPixels();

 		// Init AVFrame for source image & final (converted image)
 		frame_source = allocate_avframe(PIX_FMT_RGB24, source_image_width, source_image_height, &bytes_source);
--- a/src/Frame.cpp
+++ b/src/Frame.cpp
@@ -199,7 +199,8 @@ tr1::shared_ptr<Magick::Image> Frame::GetWaveform(int width, int height)
 		}

 		// Create image
-		wave_image = tr1::shared_ptr<Magick::Image>(new Magick::Image(Magick::Geometry(total_width, total_height), Magick::Color("#000000")));
+		wave_image = tr1::shared_ptr<Magick::Image>(new Magick::Image(Magick::Geometry(total_width, total_height), Magick::Color("none")));
+		wave_image->backgroundColor(Magick::Color("none"));

 		// Draw the waveform
 		wave_image->draw(lines);
@@ -216,7 +217,8 @@ tr1::shared_ptr<Magick::Image> Frame::GetWaveform(int width, int height)
 	else
 	{
 		// No audio samples present
-		wave_image = tr1::shared_ptr<Magick::Image>(new Magick::Image(Magick::Geometry(width, height), Magick::Color("#000000")));
+		wave_image = tr1::shared_ptr<Magick::Image>(new Magick::Image(Magick::Geometry(width, height), Magick::Color("none")));
+		wave_image->backgroundColor(Magick::Color("none"));

 		// Add Channel Label
 		lines.push_back(Magick::DrawableStrokeColor("#ffffff"));
@@ -524,19 +526,27 @@ void Frame::AddImage(tr1::shared_ptr<Magick::Image> new_image, float alpha)
 }

 // Add audio samples to a specific channel
-void Frame::AddAudio(int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f)
+void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f)
 {
 	// Extend audio buffer (if needed)
 	if (destStartSample + numSamples > audio->getNumSamples())
 		audio->setSize(audio->getNumChannels(), destStartSample + numSamples, true, true, false);

 	// Always clear the range of samples first
-	audio->clear(destChannel, destStartSample, numSamples);
+	if (replaceSamples)
+		audio->clear(destChannel, destStartSample, numSamples);

 	// Add samples to frame's audio buffer
 	audio->addFrom(destChannel, destStartSample, source, numSamples, gainToApplyToSource);
 }

+// Apply gain ramp (i.e. fading volume)
+void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
+{
+	// Apply gain ramp
+	audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
+}
+
 // Experimental method to add effects to this frame
 void Frame::AddEffect(string name)
 {
--- a/src/FrameMapper.cpp
+++ b/src/FrameMapper.cpp
@@ -288,7 +288,7 @@ tr1::shared_ptr<Frame> FrameMapper::GetFrame(int requested_frame) throw(ReaderCl
 					number_to_copy = remaining_samples;

 				// Add samples to new frame
-				frame->AddAudio(channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.Samples.sample_start, number_to_copy, 1.0);
+				frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel) + mapped.Samples.sample_start, number_to_copy, 1.0);
 			}
 			else if (starting_frame > mapped.Samples.frame_start && starting_frame < mapped.Samples.frame_end)
 			{
@@ -298,7 +298,7 @@ tr1::shared_ptr<Frame> FrameMapper::GetFrame(int requested_frame) throw(ReaderCl
 					number_to_copy = remaining_samples;

 				// Add samples to new frame
-				frame->AddAudio(channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
+				frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
 			}
 			else
 			{
@@ -308,7 +308,7 @@ tr1::shared_ptr<Frame> FrameMapper::GetFrame(int requested_frame) throw(ReaderCl
 					number_to_copy = remaining_samples;

 				// Add samples to new frame
-				frame->AddAudio(channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
+				frame->AddAudio(true, channel, samples_copied, original_frame->GetAudioSamples(channel), number_to_copy, 1.0);
 			}
 		}

--- a/src/Main.cpp
+++ b/src/Main.cpp
@@ -60,16 +60,24 @@ int main()
 	//Clip c1(new FFmpegReader("/home/jonathan/Apps/videcho_site/media/user_files/videos/bd0bf442-3221-11e2-8bf6-001fd00ee3aa.webm"));
 	Clip c1(new FFmpegReader("/home/jonathan/Music/lonely island/B004YRCAIU_(disc_1)_02_-_Mama_[Explicit].mp3"));
 	//Clip c1(new FFmpegReader("/home/jonathan/sintel.webm"));
-	Clip c2(new ImageReader("/home/jonathan/Desktop/logo1.png"));
-	Clip c3(new ImageReader("/home/jonathan/Desktop/logo1.png"));
+	Clip c2(new ImageReader("/home/jonathan/Desktop/openshot_style1.png"));
+	Clip c3(new FFmpegReader("/home/jonathan/Music/lonely island/B004YRCAUI_(disc_1)_16_-_Japan.mp3"));
 	//Clip c3(new FFmpegReader("/home/jonathan/Desktop/IncognitoCory_-_April_Song.mp3"));
 	c1.Position(0.0);
 	c1.gravity = GRAVITY_CENTER;
 	c1.scale = SCALE_FIT;
 	c1.End(20);
+	c1.Layer(2);
+	c1.volume.AddPoint(1, 0.0);
+	c1.volume.AddPoint(100, 0.5);
+	c1.volume.AddPoint(200, 0.0);
+	c1.volume.AddPoint(300, 0.5);
+	c1.Waveform(true);
+	//c1.scale_x.AddPoint(1, 0.5, LINEAR);
+	//c1.scale_y.AddPoint(1, 0.5, LINEAR);

 	c2.Position(0.0);
-	c2.Layer(2);
+	c2.Layer(1);
 	c2.gravity = GRAVITY_BOTTOM_RIGHT;
 	c2.scale = SCALE_NONE;
 	c2.alpha.AddPoint(1, 1);
@@ -79,9 +87,10 @@ int main()
 	c2.location_y.AddPoint(1, -0.02);
 	c2.End(20);

-	c3.Layer(3);
+	c3.Layer(0);
 	c3.End(20);
 	c3.gravity = GRAVITY_CENTER;
+	c3.volume.AddPoint(1, 0.5);
 	//c3.scale_x.AddPoint(1, 0.1);
 	//c3.scale_x.AddPoint(300, 2.0);
 	//c3.scale_y.AddPoint(1, 0.1);
@@ -162,8 +171,8 @@ int main()

 	// Add clips
 	t.AddClip(&c1);
-	//t.AddClip(&c2);
-	//t.AddClip(&c3);
+	t.AddClip(&c2);
+	t.AddClip(&c3);


 	// Create a writer
@@ -172,7 +181,7 @@ int main()

 	// Set options
 	//w.SetAudioOptions(true, "libmp3lame", 44100, 2, 128000, false);
-	w.SetAudioOptions(true, "libvorbis", 44100, 2, 128000, true);
+	w.SetAudioOptions(true, "libvorbis", 44100, 2, 128000);
 	w.SetVideoOptions(true, "libvpx", Fraction(24, 1), 624, 348, Fraction(1,1), false, false, 2000000);

 	// Prepare Streams
@@ -184,7 +193,7 @@ int main()
 	// Output stream info
 	w.OutputStreamInfo();

-	for (int frame = 1; frame <= 600; frame++)
+	for (int frame = 1; frame <= 300; frame++)
 	{
 		tr1::shared_ptr<Frame> f = t.GetFrame(frame);
 		if (f)
--- a/src/Timeline.cpp
+++ b/src/Timeline.cpp
@@ -56,19 +56,44 @@ void Timeline::add_layer(tr1::shared_ptr<Frame> new_frame, Clip* source_clip, in
 	tr1::shared_ptr<Frame> source_frame;
 	#pragma omp critical (reader_lock)
 		source_frame = tr1::shared_ptr<Frame>(source_clip->GetFrame(clip_frame_number));
-	tr1::shared_ptr<Magick::Image> source_image = source_frame->GetImage();
-
-	// Get some basic image properties
-	int source_width = source_image->columns();
-	int source_height = source_image->rows();
+	tr1::shared_ptr<Magick::Image> source_image;

 	/* CREATE BACKGROUND COLOR - needed if this is the 1st layer */
 	if (new_frame->GetImage()->columns() == 1)
 		new_frame->AddColor(width, height, "#000000");

-	/* COPY AUDIO */
+	/* COPY AUDIO - with correct volume */
 	for (int channel = 0; channel < source_frame->GetAudioChannelsCount(); channel++)
-		new_frame->AddAudio(channel, 0, source_frame->GetAudioSamples(channel), source_frame->GetAudioSamplesCount(), 1.0f);
+	{
+		float initial_volume = 1.0f;
+		float previous_volume = source_clip->volume.GetValue(clip_frame_number - 1); // previous frame's percentage of volume (0 to 1)
+		float volume = source_clip->volume.GetValue(clip_frame_number); // percentage of volume (0 to 1)
+
+		// If no ramp needed, set initial volume = clip's volume
+		if (isEqual(previous_volume, volume))
+			initial_volume = volume;
+
+		// Apply ramp to source frame (if needed)
+		if (!isEqual(previous_volume, volume))
+			source_frame->ApplyGainRamp(channel, 0, source_frame->GetAudioSamplesCount(), previous_volume, volume);
+
+		// Copy audio samples (and set initial volume).  Mix samples with existing audio samples.  The gains are added together, to
+		// be sure to set the gain's correctly, so the sum does not exceed 1.0 (of audio distortion will happen).
+		new_frame->AddAudio(false, channel, 0, source_frame->GetAudioSamples(channel), source_frame->GetAudioSamplesCount(), initial_volume);
+
+	}
+
+	/* GET IMAGE DATA - OR GENERATE IT */
+	if (!source_clip->Waveform())
+		// Get actual frame image data
+		source_image = source_frame->GetImage();
+	else
+		// Generate Waveform Dynamically (the size of the timeline)
+		source_image = source_frame->GetWaveform(width, height);
+
+	// Get some basic image properties
+	int source_width = source_image->columns();
+	int source_height = source_image->rows();

 	/* ALPHA & OPACITY */
 	if (source_clip->alpha.GetValue(clip_frame_number) != 0)