diff --git a/include/FFmpegReader.h b/include/FFmpegReader.h index 9ab69c73..02d357fb 100644 --- a/include/FFmpegReader.h +++ b/include/FFmpegReader.h @@ -104,7 +104,6 @@ namespace openshot AVPicture *pFrame; bool is_open; bool is_duration_known; - bool check_interlace; bool check_fps; @@ -116,7 +115,7 @@ namespace openshot map packets; map frames; map processing_video_frames; - map processing_audio_frames; + multimap processing_audio_frames; map processed_video_frames; map processed_audio_frames; AudioLocation previous_packet_location; diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 5afcfbb2..59a4f2d3 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -871,10 +871,6 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int Cache *my_cache = &working_cache; AVPacket *my_packet = packets[packet]; - // Add audio frame to list of processing audio frames - #pragma omp critical (processing_list) - processing_audio_frames[target_frame] = target_frame; - // Track 1st audio packet after a successful seek if (!seek_audio_frame_found && is_seeking) seek_audio_frame_found = target_frame; @@ -888,8 +884,6 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int AVFrame *audio_frame = avcodec_alloc_frame(); avcodec_get_frame_defaults(audio_frame); - // Allocate audio buffer - int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE]; int packet_samples = 0; int data_size = 0; @@ -915,7 +909,6 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int packet_samples = audio_frame->nb_samples * aCodecCtx->channels; } - // Estimate the # of samples and the end of this packet's location (to prevent GAPS for the next timestamp) int pts_remaining_samples = packet_samples / info.channels; // Adjust for zero based array @@ -937,6 +930,9 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int pts_counter++; prev_samples = pts_remaining_samples; + // Add audio frame to list of processing audio frames + #pragma omp critical (processing_list) + processing_audio_frames.insert(pair(previous_packet_location.frame, previous_packet_location.frame)); while (pts_remaining_samples) { @@ -955,25 +951,29 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int // next frame previous_packet_location.frame++; previous_packet_location.sample_start = 0; + + // Add audio frame to list of processing audio frames + #pragma omp critical (processing_list) + processing_audio_frames.insert(pair(previous_packet_location.frame, previous_packet_location.frame)); + } else { // Increment sample start previous_packet_location.sample_start += samples; } - } - #pragma omp critical (packet_cache) - RemoveAVPacket(my_packet); - // TODO: Disable OpenMP on audio packet processing. It is not currently possible to reassemble the packets - // in order without creating small gaps and/or overlapping sample values. - #pragma xxx omp task firstprivate(requested_frame, target_frame, my_cache, starting_sample, audio_buf) + + // Process the audio samples in a separate thread (this includes resampling to 16 bit integer, and storing + // in a openshot::Frame object). + #pragma omp task firstprivate(requested_frame, target_frame, my_cache, starting_sample, my_packet, audio_frame) { + // Allocate audio buffer + int16_t *audio_buf = new int16_t[AVCODEC_MAX_AUDIO_FRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE]; #pragma omp critical (debug_output) AppendDebugMethod("FFmpegReader::ProcessAudioPacket (ReSample)", "packet_samples", packet_samples, "info.channels", info.channels, "info.sample_rate", info.sample_rate, "aCodecCtx->sample_fmt", aCodecCtx->sample_fmt, "AV_SAMPLE_FMT_S16", AV_SAMPLE_FMT_S16, "", -1); - // Create output frame AVFrame *audio_converted = avcodec_alloc_frame(); avcodec_get_frame_defaults(audio_converted); @@ -1016,7 +1016,6 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int avcodec_free_frame(&audio_converted); - int starting_frame_number = -1; bool partial_frame = true; for (int channel_filter = 0; channel_filter < info.channels; channel_filter++) @@ -1069,32 +1068,29 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int if (samples > remaining_samples) samples = remaining_samples; - // Add audio frame to list of processing audio frames - #pragma omp critical (processing_list) - processing_audio_frames[starting_frame_number] = starting_frame_number; - tr1::shared_ptr f; #pragma omp critical (openshot_cache) // Create or get frame object f = CreateFrame(starting_frame_number); - // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent - // some louder samples from maxing out at 1.0 (not sure why this happens) - f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f); - // Determine if this frame was "partially" filled in if (samples_per_frame == start + samples) partial_frame = false; else partial_frame = true; + // Add samples for current channel to the frame. Reduce the volume to 98%, to prevent + // some louder samples from maxing out at 1.0 (not sure why this happens) + #pragma omp critical (openshot_adding_audio) + f->AddAudio(true, channel_filter, start, iterate_channel_buffer, samples, 0.98f); + // Debug output #pragma omp critical (debug_output) AppendDebugMethod("FFmpegReader::ProcessAudioPacket (f->AddAudio)", "frame", starting_frame_number, "start", start, "samples", samples, "channel", channel_filter, "partial_frame", partial_frame, "samples_per_frame", samples_per_frame); + // Add or update cache #pragma omp critical (openshot_cache) - // Add or update cache - my_cache->Add(f->number, f); + my_cache->Add(f->number, f); // Decrement remaining samples remaining_samples -= samples; @@ -1120,25 +1116,36 @@ void FFmpegReader::ProcessAudioPacket(int requested_frame, int target_frame, int delete[] audio_buf; audio_buf = NULL; - // Add video frame to list of processing video frames + // Remove audio frame from list of processing audio frames #pragma omp critical (processing_list) { // Update all frames as completed for (int f = target_frame; f < starting_frame_number; f++) { - if (f == (starting_frame_number - 1) && partial_frame) - // ignore partial frames (always the last frame processed) - break; - processing_audio_frames.erase(f); - processed_audio_frames[f] = f; + // Remove the frame # from the processing list. NOTE: If more than one thread is + // processing this frame, the frame # will be in this list multiple times. We are only + // removing a single instance of it here. + processing_audio_frames.erase(processing_audio_frames.find(f)); + + // Check and see if this frame is also being processed by another thread + if (processing_audio_frames.count(f) == 0) + // No other thread is processing it. Mark the audio as processed (final) + processed_audio_frames[f] = f; } } + #pragma omp critical (packet_cache) + RemoveAVPacket(my_packet); + // Debug output #pragma omp critical (debug_output) - AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number, "", -1, "", -1, "", -1); + AppendDebugMethod("FFmpegReader::ProcessAudioPacket (After)", "requested_frame", requested_frame, "starting_frame", target_frame, "end_frame", starting_frame_number - 1, "", -1, "", -1, "", -1); } // end task + + // TODO: Fix this bug. Wait on the task to complete. This is not ideal, but solves an issue with the + // audio_frame being modified by the next call to this method. I think this is a scope issue with OpenMP. + #pragma omp taskwait } diff --git a/src/Frame.cpp b/src/Frame.cpp index ae58065c..cdc7759b 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -834,7 +834,7 @@ void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, int new_channel_length = audio->getNumChannels(); if (destChannel >= new_channel_length) new_channel_length = destChannel + 1; - if (new_length > audio->getNumSamples() || new_channel_length >= audio->getNumChannels()) + if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels()) audio->setSize(new_channel_length, new_length, true, true, false); // Clear the range of samples first (if needed) diff --git a/src/examples/Example.cpp b/src/examples/Example.cpp index 02e59e6d..0aea3486 100644 --- a/src/examples/Example.cpp +++ b/src/examples/Example.cpp @@ -59,7 +59,7 @@ int main(int argc, char* argv[]) //ImageWriter w9("/home/jonathan/output.gif"); // Set options - w9.SetAudioOptions(true, "libmp3lame", map.info.sample_rate, map.info.channels, map.info.channel_layout, 120000); + w9.SetAudioOptions(true, "libmp3lame", r9.info.sample_rate, r9.info.channels, r9.info.channel_layout, 120000); //w9.SetAudioOptions(true, "libmp3lame", 44100, r9.info.channels, r9.info.channel_layout, 120000); //w9.SetVideoOptions(true, "libvpx", map.info.fps, map.info.width, map.info.height, map.info.pixel_ratio, false, false, 1500000); //w9.SetVideoOptions(true, "rawvideo", r9.info.fps, 400, 2, r9.info.pixel_ratio, false, false, 20000000); @@ -93,10 +93,11 @@ int main(int argc, char* argv[]) int frame_number = ( frame); cout << "get " << frame << " (frame: " << frame_number << ") " << endl; - tr1::shared_ptr f = map.GetFrame(frame_number); - //cout << "display it (" << f->number << ", " << f << ")" << endl; + tr1::shared_ptr f = r9.GetFrame(frame_number); + cout << "display it (" << f->number << ", " << f << ")" << endl; //r9.GetFrame(frame_number)->DisplayWaveform(); - //f->DisplayWaveform(); + //if (frame == 49) + // f->DisplayWaveform(); //f->AddColor(r9.info.width, r9.info.height, "blue"); w9.WriteFrame(f);