From 93c1e2eb4910229cde0de8fb130e42e34c325bee Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 25 Jul 2018 02:24:01 -0500 Subject: [PATCH 1/3] Fixing bitrate calculation (to be in bytes instead of bits), and adding in FPS detection for files which don't have valid FPS. In those cases (streaming files for example), we iterate through all packets, and average the # of frames, duration, bit rate, etc... Not idealy, but a better fallback. --- src/FFmpegReader.cpp | 112 ++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 71 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 4d435f37..e1e50594 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -339,19 +339,21 @@ void FFmpegReader::UpdateAudioInfo() void FFmpegReader::UpdateVideoInfo() { + if (check_fps) + // Already initialized all the video metadata, no reason to do it again + return; + // Set values of FileInfo struct info.has_video = true; info.file_size = pFormatCtx->pb ? avio_size(pFormatCtx->pb) : -1; info.height = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->height; info.width = AV_GET_CODEC_ATTRIBUTES(pStream, pCodecCtx)->width; info.vcodec = pCodecCtx->codec->name; - info.video_bit_rate = pFormatCtx->bit_rate; - if (!check_fps) - { - // set frames per second (fps) - info.fps.num = pStream->avg_frame_rate.num; - info.fps.den = pStream->avg_frame_rate.den; - } + info.video_bit_rate = (pFormatCtx->bit_rate / 8); + + // set frames per second (fps) + info.fps.num = pStream->avg_frame_rate.num; + info.fps.den = pStream->avg_frame_rate.den; if (pStream->sample_aspect_ratio.num != 0) { @@ -415,16 +417,22 @@ void FFmpegReader::UpdateVideoInfo() } // Override an invalid framerate - if (info.fps.ToFloat() > 240.0f || (info.fps.num == 0 || info.fps.den == 0)) - { - // Set a few important default video settings (so audio can be divided into frames) - info.fps.num = 24; - info.fps.den = 1; - info.video_timebase.num = 1; - info.video_timebase.den = 24; + if (info.fps.ToFloat() > 240.0f || (info.fps.num == 0 || info.fps.den == 0)) { + // Calculate FPS, duration, video bit rate, and video length manually + // by scanning through all the video stream packets + CheckFPS(); - // Calculate number of frames - info.video_length = round(info.duration * info.fps.ToDouble()); + // If still an invalid FPS detected, just default to 24 FPS + // Set a few important default video settings (so audio can be divided into frames) + if (info.fps.ToFloat() > 240.0f || (info.fps.num == 0 || info.fps.den == 0)) { + info.fps.num = 24; + info.fps.den = 1; + info.video_timebase.num = 1; + info.video_timebase.den = 24; + + // Calculate number of frames + info.video_length = round(info.duration * info.fps.ToDouble()); + } } // Add video metadata (if any) @@ -1857,16 +1865,12 @@ void FFmpegReader::CheckWorkingFrames(bool end_of_stream, int64_t requested_fram void FFmpegReader::CheckFPS() { check_fps = true; - AV_ALLOCATE_IMAGE(pFrame, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), info.width, info.height); - int first_second_counter = 0; int second_second_counter = 0; int third_second_counter = 0; int forth_second_counter = 0; int fifth_second_counter = 0; - - int iterations = 0; - int threshold = 500; + int frames_detected = 0; // Loop through the stream while (true) @@ -1908,63 +1912,29 @@ void FFmpegReader::CheckFPS() forth_second_counter++; else if (video_seconds > 4.0 && video_seconds <= 5.0) fifth_second_counter++; - else - // Too far - break; + + // Increment counters + frames_detected++; } } - - // Increment counters - iterations++; - - // Give up (if threshold exceeded) - if (iterations > threshold) - break; } // Double check that all counters have greater than zero (or give up) - if (second_second_counter == 0 || third_second_counter == 0 || forth_second_counter == 0 || fifth_second_counter == 0) - { - // Seek to frame 1 - Seek(1); + if (second_second_counter != 0 && third_second_counter != 0 && forth_second_counter != 0 && fifth_second_counter != 0) { + // Calculate average FPS + int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter; + int avg_fps = round(sum_fps / 4.0f); - // exit with no changes to FPS (not enough data to calculate) - return; + // Update FPS + info.fps = Fraction(avg_fps, 1); + + // Update Duration and Length + info.video_length = frames_detected; + info.duration = frames_detected / round(sum_fps / 4.0f); + + // Update video bit rate + info.video_bit_rate = info.file_size / info.duration; } - - int sum_fps = second_second_counter + third_second_counter + forth_second_counter + fifth_second_counter; - int avg_fps = round(sum_fps / 4.0f); - - // Sometimes the FPS is incorrectly detected by FFmpeg. If the 1st and 2nd seconds counters - // agree with each other, we are going to adjust the FPS of this reader instance. Otherwise, print - // a warning message. - - // Get diff from actual frame rate - double fps = info.fps.ToDouble(); - double diff = fps - double(avg_fps); - - // Is difference bigger than 1 frame? - if (diff <= -1 || diff >= 1) - { - // Compare to half the frame rate (the most common type of issue) - double half_fps = Fraction(info.fps.num / 2, info.fps.den).ToDouble(); - diff = half_fps - double(avg_fps); - - // Is difference bigger than 1 frame? - if (diff <= -1 || diff >= 1) - { - // Update FPS for this reader instance - info.fps = Fraction(avg_fps, 1); - } - else - { - // Update FPS for this reader instance (to 1/2 the original framerate) - info.fps = Fraction(info.fps.num / 2, info.fps.den); - } - } - - // Seek to frame 1 - Seek(1); } // Remove AVFrame from cache (and deallocate it's memory) From 435932f41569c70a5741ff65522b4a6876ab6599 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 2 Aug 2018 00:40:44 -0500 Subject: [PATCH 2/3] Fixing another issue where larger FPS files are incorrectly changed to a different FPS --- src/FFmpegReader.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index e1e50594..0b100050 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -421,18 +421,6 @@ void FFmpegReader::UpdateVideoInfo() // Calculate FPS, duration, video bit rate, and video length manually // by scanning through all the video stream packets CheckFPS(); - - // If still an invalid FPS detected, just default to 24 FPS - // Set a few important default video settings (so audio can be divided into frames) - if (info.fps.ToFloat() > 240.0f || (info.fps.num == 0 || info.fps.den == 0)) { - info.fps.num = 24; - info.fps.den = 1; - info.video_timebase.num = 1; - info.video_timebase.den = 24; - - // Calculate number of frames - info.video_length = round(info.duration * info.fps.ToDouble()); - } } // Add video metadata (if any) @@ -1934,6 +1922,18 @@ void FFmpegReader::CheckFPS() // Update video bit rate info.video_bit_rate = info.file_size / info.duration; + } else { + + // Too short to determine framerate, just default FPS + // Set a few important default video settings (so audio can be divided into frames) + info.fps.num = 30; + info.fps.den = 1; + info.video_timebase.num = info.fps.den; + info.video_timebase.den = info.fps.num; + + // Calculate number of frames + info.video_length = frames_detected; + info.duration = frames_detected / info.video_timebase.ToFloat(); } } From da01a2c4cb1b29b701bc014b74353081f464daa4 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 2 Aug 2018 00:42:14 -0500 Subject: [PATCH 3/3] Adding "reader" property for Mask effect, to allow the user to adjust the image or video used by the mask effect. --- include/effects/Mask.h | 1 + src/effects/Mask.cpp | 76 ++++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/include/effects/Mask.h b/include/effects/Mask.h index ad1a6aab..ef707f5f 100644 --- a/include/effects/Mask.h +++ b/include/effects/Mask.h @@ -65,6 +65,7 @@ namespace openshot private: ReaderBase *reader; std::shared_ptr original_mask; + bool needs_refresh; /// Init effect settings void init_effect_details(); diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 58f00d20..f8f34ac6 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -30,14 +30,14 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Mask::Mask() : reader(NULL), replace_image(false) { +Mask::Mask() : reader(NULL), replace_image(false), needs_refresh(true) { // Init effect properties init_effect_details(); } // Default constructor Mask::Mask(ReaderBase *mask_reader, Keyframe mask_brightness, Keyframe mask_contrast) : - reader(mask_reader), brightness(mask_brightness), contrast(mask_contrast), replace_image(false) + reader(mask_reader), brightness(mask_brightness), contrast(mask_contrast), replace_image(false), needs_refresh(true) { // Init effect properties init_effect_details(); @@ -77,7 +77,7 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr frame, int64_t fram // Get mask image (if missing or different size than frame image) #pragma omp critical (open_mask_reader) { - if (!original_mask || !reader->info.has_single_image || + if (!original_mask || !reader->info.has_single_image || needs_refresh || (original_mask && original_mask->size() != frame_image->size())) { // Only get mask if needed @@ -91,6 +91,9 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr frame, int64_t fram } } + // Refresh no longer needed + needs_refresh = false; + // Get pixel arrays unsigned char *pixels = (unsigned char *) frame_image->bits(); unsigned char *mask_pixels = (unsigned char *) original_mask->bits(); @@ -206,47 +209,51 @@ void Mask::SetJsonValue(Json::Value root) { contrast.SetJsonValue(root["contrast"]); if (!root["reader"].isNull()) // does Json contain a reader? { - - if (!root["reader"]["type"].isNull()) // does the reader Json contain a 'type'? + #pragma omp critical (open_mask_reader) { - // Close previous reader (if any) - if (reader) + // This reader has changed, so refresh cached assets + needs_refresh = true; + + if (!root["reader"]["type"].isNull()) // does the reader Json contain a 'type'? { - // Close and delete existing reader (if any) - reader->Close(); - delete reader; - reader = NULL; - } + // Close previous reader (if any) + if (reader) { + // Close and delete existing reader (if any) + reader->Close(); + delete reader; + reader = NULL; + } - // Create new reader (and load properties) - string type = root["reader"]["type"].asString(); + // Create new reader (and load properties) + string type = root["reader"]["type"].asString(); - if (type == "FFmpegReader") { + if (type == "FFmpegReader") { - // Create new reader - reader = new FFmpegReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); + // Create new reader + reader = new FFmpegReader(root["reader"]["path"].asString()); + reader->SetJsonValue(root["reader"]); -#ifdef USE_IMAGEMAGICK - } else if (type == "ImageReader") { + #ifdef USE_IMAGEMAGICK + } else if (type == "ImageReader") { - // Create new reader - reader = new ImageReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); -#endif + // Create new reader + reader = new ImageReader(root["reader"]["path"].asString()); + reader->SetJsonValue(root["reader"]); + #endif - } else if (type == "QtImageReader") { + } else if (type == "QtImageReader") { - // Create new reader - reader = new QtImageReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); + // Create new reader + reader = new QtImageReader(root["reader"]["path"].asString()); + reader->SetJsonValue(root["reader"]); - } else if (type == "ChunkReader") { + } else if (type == "ChunkReader") { - // Create new reader - reader = new ChunkReader(root["reader"]["path"].asString(), (ChunkVersion) root["reader"]["chunk_version"].asInt()); - reader->SetJsonValue(root["reader"]); + // Create new reader + reader = new ChunkReader(root["reader"]["path"].asString(), (ChunkVersion) root["reader"]["chunk_version"].asInt()); + reader->SetJsonValue(root["reader"]); + } } } @@ -275,6 +282,11 @@ string Mask::PropertiesJSON(int64_t requested_frame) { root["brightness"] = add_property_json("Brightness", brightness.GetValue(requested_frame), "float", "", &brightness, -1.0, 1.0, false, requested_frame); root["contrast"] = add_property_json("Contrast", contrast.GetValue(requested_frame), "float", "", &contrast, 0, 20, false, requested_frame); + if (reader) + root["reader"] = add_property_json("Source", 0.0, "reader", reader->Json(), NULL, 0, 1, false, requested_frame); + else + root["reader"] = add_property_json("Source", 0.0, "reader", "{}", NULL, 0, 1, false, requested_frame); + // Return formatted string return root.toStyledString(); }