From d87e9e49c8d3f8dc2657eedb102e69cc7c3aaa1c Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 10 Feb 2024 20:28:47 -0600 Subject: [PATCH] Un-reverting previous commit, removing queuing from FFmpegWriter for simplicity. Increasing margin on webm unit tests, when checking pixel color. --- src/FFmpegWriter.cpp | 173 ++++++++++++----------------------------- src/FFmpegWriter.h | 21 +---- tests/FFmpegWriter.cpp | 8 +- 3 files changed, 55 insertions(+), 147 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 273afdb4..eea37274 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -75,7 +75,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6 FFmpegWriter::FFmpegWriter(const std::string& path) : path(path), oc(NULL), audio_st(NULL), video_st(NULL), samples(NULL), audio_outbuf(NULL), audio_outbuf_size(0), audio_input_frame_size(0), audio_input_position(0), - initial_audio_input_frame_size(0), img_convert_ctx(NULL), cache_size(1), num_of_rescalers(1), + initial_audio_input_frame_size(0), img_convert_ctx(NULL), num_of_rescalers(1), rescaler_position(0), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0), original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false), write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) { @@ -679,118 +679,50 @@ void FFmpegWriter::WriteFrame(std::shared_ptr frame) { if (!is_open) throw WriterClosed("The FFmpegWriter is closed. Call Open() before calling this method.", path); - // Add frame pointer to "queue", waiting to be processed the next - // time the WriteFrames() method is called. - if (info.has_video && video_st) - spooled_video_frames.push_back(frame); - - if (info.has_audio && audio_st) - spooled_audio_frames.push_back(frame); - ZmqLogger::Instance()->AppendDebugMethod( "FFmpegWriter::WriteFrame", "frame->number", frame->number, - "spooled_video_frames.size()", spooled_video_frames.size(), - "spooled_audio_frames.size()", spooled_audio_frames.size(), - "cache_size", cache_size, "is_writing", is_writing); - // Write the frames once it reaches the correct cache size - if ((int)spooled_video_frames.size() == cache_size || (int)spooled_audio_frames.size() == cache_size) { - // Write frames to video file - write_queued_frames(); - } + // Write frames to video file + write_frame(frame); // Keep track of the last frame added last_frame = frame; } // Write all frames in the queue to the video file. -void FFmpegWriter::write_queued_frames() { - ZmqLogger::Instance()->AppendDebugMethod( - "FFmpegWriter::write_queued_frames", - "spooled_video_frames.size()", spooled_video_frames.size(), - "spooled_audio_frames.size()", spooled_audio_frames.size()); - +void FFmpegWriter::write_frame(std::shared_ptr frame) { // Flip writing flag is_writing = true; - // Transfer spool to queue - queued_video_frames = spooled_video_frames; - queued_audio_frames = spooled_audio_frames; - - // Empty spool - spooled_video_frames.clear(); - spooled_audio_frames.clear(); - // Create blank exception bool has_error_encoding_video = false; - // Process all audio frames (in a separate thread) - if (info.has_audio && audio_st && !queued_audio_frames.empty()) - write_audio_packets(false); + // Process audio frame + if (info.has_audio && audio_st) + write_audio_packets(false, frame); - // Loop through each queued image frame - while (!queued_video_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = queued_video_frames.front(); - - // Add to processed queue - processed_frames.push_back(frame); - - // Encode and add the frame to the output file - if (info.has_video && video_st) - process_video_packet(frame); - - // Remove front item - queued_video_frames.pop_front(); - - } // end while - - - // Loop back through the frames (in order), and write them to the video file - while (!processed_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = processed_frames.front(); - - if (info.has_video && video_st) { - // Add to deallocate queue (so we can remove the AVFrames when we are done) - deallocate_frames.push_back(frame); - - // Does this frame's AVFrame still exist - if (av_frames.count(frame)) { - // Get AVFrame - AVFrame *frame_final = av_frames[frame]; - - // Write frame to video file - bool success = write_video_packet(frame, frame_final); - if (!success) - has_error_encoding_video = true; - } - } - - // Remove front item - processed_frames.pop_front(); - } - - // Loop through, and deallocate AVFrames - while (!deallocate_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = deallocate_frames.front(); + // Process video frame + if (info.has_video && video_st) + process_video_packet(frame); + if (info.has_video && video_st) { // Does this frame's AVFrame still exist if (av_frames.count(frame)) { // Get AVFrame - AVFrame *av_frame = av_frames[frame]; + AVFrame *frame_final = av_frames[frame]; + + // Write frame to video file + if (!write_video_packet(frame, frame_final)) { + has_error_encoding_video = true; + } // Deallocate buffer and AVFrame - av_freep(&(av_frame->data[0])); - AV_FREE_FRAME(&av_frame); + av_freep(&(frame_final->data[0])); + AV_FREE_FRAME(&frame_final); av_frames.erase(frame); } - - // Remove front item - deallocate_frames.pop_front(); } // Done writing @@ -820,12 +752,9 @@ void FFmpegWriter::WriteFrame(ReaderBase *reader, int64_t start, int64_t length) // Write the file trailer (after all frames are written) void FFmpegWriter::WriteTrailer() { - // Write any remaining queued frames to video file - write_queued_frames(); - // Process final audio frame (if any) if (info.has_audio && audio_st) - write_audio_packets(true); + write_audio_packets(true, NULL); // Flush encoders (who sometimes hold on to frames) flush_encoders(); @@ -1598,7 +1527,10 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { } // write all queued frames' audio to the video file -void FFmpegWriter::write_audio_packets(bool is_final) { +void FFmpegWriter::write_audio_packets(bool is_final, std::shared_ptr frame) { + if (!frame && !is_final) + return; + // Init audio buffers / variables int total_frame_samples = 0; int frame_position = 0; @@ -1608,56 +1540,49 @@ void FFmpegWriter::write_audio_packets(bool is_final) { ChannelLayout channel_layout_in_frame = LAYOUT_MONO; // default channel layout // Create a new array (to hold all S16 audio samples, for the current queued frames - unsigned int all_queued_samples_size = sizeof(int16_t) * (queued_audio_frames.size() * AVCODEC_MAX_AUDIO_FRAME_SIZE); + unsigned int all_queued_samples_size = sizeof(int16_t) * AVCODEC_MAX_AUDIO_FRAME_SIZE; int16_t *all_queued_samples = (int16_t *) av_malloc(all_queued_samples_size); int16_t *all_resampled_samples = NULL; int16_t *final_samples_planar = NULL; int16_t *final_samples = NULL; - // Loop through each queued audio frame - while (!queued_audio_frames.empty()) { - // Get front frame (from the queue) - std::shared_ptr frame = queued_audio_frames.front(); + // Get audio sample array + float *frame_samples_float = NULL; - // Get the audio details from this frame + // Get the audio details from this frame + if (frame) { sample_rate_in_frame = frame->SampleRate(); samples_in_frame = frame->GetAudioSamplesCount(); channels_in_frame = frame->GetAudioChannelsCount(); channel_layout_in_frame = frame->ChannelsLayout(); - // Get audio sample array - float *frame_samples_float = NULL; // Get samples interleaved together (c1 c2 c1 c2 c1 c2) frame_samples_float = frame->GetInterleavedAudioSamples(&samples_in_frame); + } - // Calculate total samples - total_frame_samples = samples_in_frame * channels_in_frame; + // Calculate total samples + total_frame_samples = samples_in_frame * channels_in_frame; - // Translate audio sample values back to 16 bit integers with saturation - const int16_t max16 = 32767; - const int16_t min16 = -32768; - for (int s = 0; s < total_frame_samples; s++, frame_position++) { - float valF = frame_samples_float[s] * (1 << 15); - int16_t conv; - if (valF > max16) { - conv = max16; - } else if (valF < min16) { - conv = min16; - } else { - conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding - } - - // Copy into buffer - all_queued_samples[frame_position] = conv; + // Translate audio sample values back to 16 bit integers with saturation + const int16_t max16 = 32767; + const int16_t min16 = -32768; + for (int s = 0; s < total_frame_samples; s++, frame_position++) { + float valF = frame_samples_float[s] * (1 << 15); + int16_t conv; + if (valF > max16) { + conv = max16; + } else if (valF < min16) { + conv = min16; + } else { + conv = int(valF + 32768.5) - 32768; // +0.5 is for rounding } - // Deallocate float array - delete[] frame_samples_float; + // Copy into buffer + all_queued_samples[frame_position] = conv; + } - // Remove front item - queued_audio_frames.pop_front(); - - } // end while + // Deallocate float array + delete[] frame_samples_float; // Update total samples (since we've combined all queued frames) diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 3f16586b..4aa89550 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -116,7 +116,6 @@ namespace openshot { class FFmpegWriter : public WriterBase { private: std::string path; - int cache_size; bool is_writing; bool is_open; int64_t video_timestamp; @@ -152,15 +151,6 @@ namespace openshot { int original_channels; std::shared_ptr last_frame; - std::deque > spooled_audio_frames; - std::deque > spooled_video_frames; - - std::deque > queued_audio_frames; - std::deque > queued_video_frames; - - std::deque > processed_frames; - std::deque > deallocate_frames; - std::map, AVFrame *> av_frames; /// Add an AVFrame to the cache @@ -205,13 +195,13 @@ namespace openshot { void process_video_packet(std::shared_ptr frame); /// write all queued frames' audio to the video file - void write_audio_packets(bool is_final); + void write_audio_packets(bool is_final, std::shared_ptr frame); /// write video frame bool write_video_packet(std::shared_ptr frame, AVFrame *frame_final); /// write all queued frames - void write_queued_frames(); + void write_frame(std::shared_ptr frame); public: @@ -224,9 +214,6 @@ namespace openshot { /// Close the writer void Close(); - /// Get the cache size (number of frames to queue before writing) - int GetCacheSize() { return cache_size; }; - /// Determine if writer is open or closed bool IsOpen() { return is_open; }; @@ -273,10 +260,6 @@ namespace openshot { /// \note This is an overloaded function. void SetAudioOptions(std::string codec, int sample_rate, int bit_rate); - /// @brief Set the cache size - /// @param new_size The number of frames to queue before writing to the file - void SetCacheSize(int new_size) { cache_size = new_size; }; - /// @brief Set video export options /// @param has_video Does this file need a video stream /// @param codec The codec used to encode the images in this video diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index cbaa7264..dcdcd1a3 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -66,10 +66,10 @@ TEST_CASE( "Webm", "[libopenshot][ffmpegwriter]" ) int pixel_index = 112 * 4; // pixel 112 (4 bytes per pixel) // Check image properties on scanline 10, pixel 112 - CHECK((int)pixels[pixel_index] == Approx(23).margin(5)); - CHECK((int)pixels[pixel_index + 1] == Approx(23).margin(5)); - CHECK((int)pixels[pixel_index + 2] == Approx(23).margin(5)); - CHECK((int)pixels[pixel_index + 3] == Approx(255).margin(5)); + CHECK((int)pixels[pixel_index] == Approx(23).margin(7)); + CHECK((int)pixels[pixel_index + 1] == Approx(23).margin(7)); + CHECK((int)pixels[pixel_index + 2] == Approx(23).margin(7)); + CHECK((int)pixels[pixel_index + 3] == Approx(255).margin(7)); } TEST_CASE( "Options_Overloads", "[libopenshot][ffmpegwriter]" )