Merge pull request #113 from OpenShot/feature/new-audio-mixing

New audio mixing code (to prevent exceeding 100%). Merging into develop.
2026-03-02 08:53:52 -08:00 · 2018-06-11 23:38:03 -07:00
parent c34ffdcd65 720c2d5ed7
commit 2a76d83e68
4 changed files with 24 additions and 49 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,9 +9,9 @@ linux-builder:
    paths:
    - build/install-x64/*
  script:
-    - "curl -f -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=linux-builder"
+    - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=linux-builder"
    - if [ ! -f artifacts.zip ]; then
-    -    "curl -f -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/develop/download?job=linux-builder"
+    -    "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/develop/download?job=linux-builder"
    - fi
    - unzip artifacts.zip
    - export LIBOPENSHOT_AUDIO_DIR=$CI_PROJECT_DIR/build/install-x64
@@ -21,7 +21,7 @@ linux-builder:
    - make install
    - cp src/bindings/python/*openshot* install-x64/lib
    - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID" > "install-x64/share/$CI_PROJECT_NAME"
-  when: always
+  when: manual
  tags:
    - linux

@@ -32,9 +32,9 @@ mac-builder:
    paths:
    - build/install-x64/*
  script:
-    - "curl -f -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=mac-builder"
+    - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=mac-builder"
    - if [ ! -f artifacts.zip ]; then
-    -    "curl -f -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/develop/download?job=mac-builder"
+    -    "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/develop/download?job=mac-builder"
    - fi
    - unzip artifacts.zip
    - export LIBOPENSHOT_AUDIO_DIR=$CI_PROJECT_DIR/build/install-x64
@@ -55,7 +55,7 @@ windows-builder-x86:
    paths:
    - build\install-x86\*
  script:
-    - Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x86" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip"
+    - try { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x86" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } catch { $_.Exception.Response.StatusCode.Value__ }
    - if (-not (Test-Path "artifacts.zip")) { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/develop/download?job=windows-builder-x86" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" }
    - Expand-Archive -Path artifacts.zip -DestinationPath .
    - $env:LIBOPENSHOT_AUDIO_DIR = "$CI_PROJECT_DIR\build\install-x86"
@@ -80,7 +80,7 @@ windows-builder-x64:
    paths:
    - build\install-x64\*
  script:
-    - Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x64" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip"
+    - try { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x64" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } catch { $_.Exception.Response.StatusCode.Value__ }
    - if (-not (Test-Path "artifacts.zip")) { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/develop/download?job=windows-builder-x64" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" }
    - Expand-Archive -Path artifacts.zip -DestinationPath .
    - $env:LIBOPENSHOT_AUDIO_DIR = "$CI_PROJECT_DIR\build\install-x64"
@@ -105,4 +105,4 @@ trigger-pipeline:
  when: always
  dependencies: []
  tags:
-    - linux
+    - gitlab
--- a/include/Timeline.h
+++ b/include/Timeline.h
@@ -153,7 +153,7 @@ namespace openshot {
 		CacheBase *final_cache; ///<Final cache of timeline frames

 		/// Process a new layer of video or audio
-		void add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, int64_t clip_frame_number, int64_t timeline_frame_number, bool is_top_clip);
+		void add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, int64_t clip_frame_number, int64_t timeline_frame_number, bool is_top_clip, float max_volume);

 		/// Apply a FrameMapper to a clip which matches the settings of this timeline
 		void apply_mapper_to_clip(Clip* clip);
--- a/src/Frame.cpp
+++ b/src/Frame.cpp
@@ -854,33 +854,8 @@ void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample,
 		if (replaceSamples)
 			audio->clear(destChannel, destStartSampleAdjusted, numSamples);

-		// Get max volume of the current audio data
-		// TODO: This always appears to be 0, which is probably not expected since that means gainFactor is always multiplied by 1.0 below.
-		// "sum_volumes = current_max_volume + new_max_volume" is then alwasy "sum_volumes = 0 + new_max_volume",
-		// which makes "gainFactor *= ((current_max_volume + new_max_volume) - (current_max_volume * new_max_volume)) / sum_volumes;"
-		// which simplifies to "gainFactor *= new_max_volume / new_max_volume;" aka "gainFactor *= 1.0"
-		// - Rich Alloway
-		float current_max_volume = audio->getMagnitude(destChannel, destStartSampleAdjusted, numSamples);
-
-		// Determine max volume of new audio data (before we add them together)
-		float new_max_volume = 0.0;
-		for (int sample=0; sample<numSamples; sample++) {
-			if (source[sample] > new_max_volume)
-				new_max_volume = source[sample];
-		}
-
-		// Determine volume adjustments (to prevent overflows)
-		float sum_volumes = current_max_volume + new_max_volume;
-		float gainFactor = gainToApplyToSource;
-		if (sum_volumes > 0.0) {
-			// Reduce both sources by this amount (existing samples and new samples)
-			gainFactor *= ((current_max_volume + new_max_volume) - (current_max_volume * new_max_volume)) / sum_volumes;
-			audio->applyGain(gainFactor);
-			ZmqLogger::Instance()->AppendDebugMethod("Frame::AddAudio", "gainToApplyToSource", gainToApplyToSource, "gainFactor", gainFactor, "sum_volumes", sum_volumes, "current_max_volume", current_max_volume, "new_max_volume", new_max_volume, "((current_max_volume + new_max_volume) - (current_max_volume * new_max_volume)) / sum_volumes", ((current_max_volume + new_max_volume) - (current_max_volume * new_max_volume)) / sum_volumes);
-		}
-
 		// Add samples to frame's audio buffer
-		audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainFactor);
+		audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
 		has_audio_data = true;

 		// Calculate max audio sample added
--- a/src/Timeline.cpp
+++ b/src/Timeline.cpp
@@ -245,7 +245,7 @@ std::shared_ptr<Frame> Timeline::GetOrCreateFrame(Clip* clip, int64_t number)
 }

 // Process a new layer of video or audio
-void Timeline::add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, int64_t clip_frame_number, int64_t timeline_frame_number, bool is_top_clip)
+void Timeline::add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, int64_t clip_frame_number, int64_t timeline_frame_number, bool is_top_clip, float max_volume)
 {
 	// Get the clip's frame & image
 	std::shared_ptr<Frame> source_frame;
@@ -288,16 +288,14 @@ void Timeline::add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, in

 	/* COPY AUDIO - with correct volume */
 	if (source_clip->Reader()->info.has_audio) {
-
 		// Debug output
 		ZmqLogger::Instance()->AppendDebugMethod("Timeline::add_layer (Copy Audio)", "source_clip->Reader()->info.has_audio", source_clip->Reader()->info.has_audio, "source_frame->GetAudioChannelsCount()", source_frame->GetAudioChannelsCount(), "info.channels", info.channels, "clip_frame_number", clip_frame_number, "timeline_frame_number", timeline_frame_number, "", -1);

 		if (source_frame->GetAudioChannelsCount() == info.channels && source_clip->has_audio.GetInt(clip_frame_number) != 0)
 			for (int channel = 0; channel < source_frame->GetAudioChannelsCount(); channel++)
 			{
-				float initial_volume = 1.0f;
-				float previous_volume = source_clip->volume.GetValue(clip_frame_number - 1); // previous frame's percentage of volume (0 to 1)
-				float volume = source_clip->volume.GetValue(clip_frame_number); // percentage of volume (0 to 1)
+				float previous_volume = source_clip->volume.GetValue(clip_frame_number - 1) / fmaxf(max_volume, 1.0); // previous frame's percentage of volume (0 to 1)
+				float volume = source_clip->volume.GetValue(clip_frame_number) / fmaxf(max_volume, 1.0); // percentage of volume (0 to 1)
 				int channel_filter = source_clip->channel_filter.GetInt(clip_frame_number); // optional channel to filter (if not -1)
 				int channel_mapping = source_clip->channel_mapping.GetInt(clip_frame_number); // optional channel to map this channel to (if not -1)

@@ -313,12 +311,8 @@ void Timeline::add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, in
 				if (channel_mapping == -1)
 					channel_mapping = channel;

-				// If no ramp needed, set initial volume = clip's volume
-				if (isEqual(previous_volume, volume))
-					initial_volume = volume;
-
 				// Apply ramp to source frame (if needed)
-				if (!isEqual(previous_volume, volume))
+				if (!isEqual(previous_volume, 1.0) || !isEqual(volume, 1.0))
 					source_frame->ApplyGainRamp(channel_mapping, 0, source_frame->GetAudioSamplesCount(), previous_volume, volume);

 				// TODO: Improve FrameMapper (or Timeline) to always get the correct number of samples per frame.
@@ -333,7 +327,7 @@ void Timeline::add_layer(std::shared_ptr<Frame> new_frame, Clip* source_clip, in
 				// Copy audio samples (and set initial volume).  Mix samples with existing audio samples.  The gains are added together, to
 				// be sure to set the gain's correctly, so the sum does not exceed 1.0 (of audio distortion will happen).
 				#pragma omp critical (T_addLayer)
-				new_frame->AddAudio(false, channel_mapping, 0, source_frame->GetAudioSamples(channel), source_frame->GetAudioSamplesCount(), initial_volume);
+				new_frame->AddAudio(false, channel_mapping, 0, source_frame->GetAudioSamples(channel), source_frame->GetAudioSamplesCount(), 1.0);

 			}
 		else
@@ -757,17 +751,23 @@ std::shared_ptr<Frame> Timeline::GetFrame(int64_t requested_frame)
 					{
 						// Determine if clip is "top" clip on this layer (only happens when multiple clips are overlapping)
 						bool is_top_clip = true;
+						float max_volume = 0.0;
 						for (int top_clip_index = 0; top_clip_index < nearby_clips.size(); top_clip_index++)
 						{
 							Clip *nearby_clip = nearby_clips[top_clip_index];
                            long nearby_clip_start_position = round(nearby_clip->Position() * info.fps.ToDouble()) + 1;
                            long nearby_clip_end_position = round((nearby_clip->Position() + nearby_clip->Duration()) * info.fps.ToDouble()) + 1;
+							long nearby_clip_start_frame = (nearby_clip->Start() * info.fps.ToDouble()) + 1;
+							long nearby_clip_frame_number = frame_number - nearby_clip_start_position + nearby_clip_start_frame;

 							if (clip->Id() != nearby_clip->Id() && clip->Layer() == nearby_clip->Layer() &&
                                    nearby_clip_start_position <= frame_number && nearby_clip_end_position >= frame_number &&
-                                    nearby_clip_start_position > clip_start_position) {
+                                    nearby_clip_start_position > clip_start_position && is_top_clip == true) {
 								is_top_clip = false;
-								break;
+							}
+
+							if (nearby_clip_start_position <= frame_number && nearby_clip_end_position >= frame_number) {
+								max_volume += nearby_clip->volume.GetValue(nearby_clip_frame_number);
 							}
 						}

@@ -779,7 +779,7 @@ std::shared_ptr<Frame> Timeline::GetFrame(int64_t requested_frame)
 						ZmqLogger::Instance()->AppendDebugMethod("Timeline::GetFrame (Calculate clip's frame #)", "clip->Position()", clip->Position(), "clip->Start()", clip->Start(), "info.fps.ToFloat()", info.fps.ToFloat(), "clip_frame_number", clip_frame_number, "", -1, "", -1);

 						// Add clip's frame as layer
-						add_layer(new_frame, clip, clip_frame_number, frame_number, is_top_clip);
+						add_layer(new_frame, clip, clip_frame_number, frame_number, is_top_clip, max_volume);

 					} else
 						// Debug output