From de012ac6c847b3b07c5b7b88000289930d58602e Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Feb 2026 16:46:40 -0600 Subject: [PATCH] Improve seek retry fallback and clean up hw decode logging - add adaptive seek fallback and switch fprintf to ZmqLogger - improves seeking on certain files by up to 30% (faster) - avoids situations where seeking too far would sometimes freeze on certain videos --- examples/Example.cpp | 149 ++++++++++++++++++++++++++----------------- src/FFmpegReader.cpp | 109 ++++++++++++++++++++++--------- src/FFmpegReader.h | 4 +- 3 files changed, 174 insertions(+), 88 deletions(-) diff --git a/examples/Example.cpp b/examples/Example.cpp index 2067fb56..0e143c3b 100644 --- a/examples/Example.cpp +++ b/examples/Example.cpp @@ -13,83 +13,116 @@ #include #include #include +#include +#include "Clip.h" #include "Frame.h" #include "FFmpegReader.h" -#include "FFmpegWriter.h" +#include "Settings.h" #include "Timeline.h" -#include "Qt/VideoCacheThread.h" // <— your new header using namespace openshot; int main(int argc, char* argv[]) { + using clock = std::chrono::high_resolution_clock; + auto total_start = clock::now(); + const std::string output_dir = "/home/jonathan/Downloads"; + const std::string input_paths[] = { + "/home/jonathan/Videos/3.4 Release/Screencasts/Timing.mp4", + "/home/jonathan/Downloads/openshot-testing/sintel_trailer-720p.mp4" + }; + const int64_t frames_to_fetch[] = {175, 225, 240, 500, 1000}; + const bool use_hw_decode = false; - // 1) Open the FFmpegReader as usual - const char* input_path = "/home/jonathan/Downloads/openshot-testing/sintel_trailer-720p.mp4"; - FFmpegReader reader(input_path); - reader.Open(); + std::cout << "Hardware decode: " << (use_hw_decode ? "ON" : "OFF") << "\n"; + openshot::Settings::Instance()->HARDWARE_DECODER = use_hw_decode ? 1 : 0; - const int64_t total_frames = reader.info.video_length; - std::cout << "Total frames: " << total_frames << "\n"; + for (const std::string& input_path : input_paths) { + auto file_start = clock::now(); + std::string base = input_path; + size_t slash = base.find_last_of('/'); + if (slash != std::string::npos) { + base = base.substr(slash + 1); + } + std::cout << "\n=== File: " << base << " ===\n"; + auto t0 = clock::now(); + FFmpegReader reader(input_path.c_str()); + auto t1 = clock::now(); + std::cout << "FFmpegReader ctor: " + << std::chrono::duration_cast(t1 - t0).count() + << " ms\n"; - Timeline timeline(reader.info.width, reader.info.height, reader.info.fps, reader.info.sample_rate, reader.info.channels, reader.info.channel_layout); - Clip c1(&reader); - timeline.AddClip(&c1); - timeline.Open(); - timeline.DisplayInfo(); + auto t2 = clock::now(); + reader.Open(); + auto t3 = clock::now(); + std::cout << "FFmpegReader Open(): " + << std::chrono::duration_cast(t3 - t2).count() + << " ms\n"; + auto t4 = clock::now(); + Timeline timeline(1920, 1080, Fraction(30, 1), reader.info.sample_rate, reader.info.channels, reader.info.channel_layout); + timeline.SetMaxSize(640, 480); + auto t5 = clock::now(); + std::cout << "Timeline ctor (1080p30): " + << std::chrono::duration_cast(t5 - t4).count() + << " ms\n"; - // 2) Construct a VideoCacheThread around 'reader' and start its background loop - // (VideoCacheThread inherits juce::Thread) - std::shared_ptr cache = std::make_shared(); - cache->Reader(&timeline); // attaches the FFmpegReader and internally calls Play() - cache->StartThread(); // juce::Thread method, begins run() + auto t6 = clock::now(); + Clip c1(&reader); + auto t7 = clock::now(); + std::cout << "Clip ctor: " + << std::chrono::duration_cast(t7 - t6).count() + << " ms\n"; - // 3) Set up the writer exactly as before - FFmpegWriter writer("/home/jonathan/Downloads/performance‐cachetest.mp4"); - writer.SetAudioOptions("aac", 48000, 192000); - writer.SetVideoOptions("libx264", 1280, 720, Fraction(30, 1), 5000000); - writer.Open(); + timeline.AddClip(&c1); - // 4) Forward pass: for each frame 1…N, tell the cache thread to seek to that frame, - // then immediately call cache->GetFrame(frame), which will block only if that frame - // hasn’t been decoded into the cache yet. - auto t0 = std::chrono::high_resolution_clock::now(); - cache->setSpeed(1); - for (int64_t f = 1; f <= total_frames; ++f) { - float pct = (float(f) / total_frames) * 100.0f; - std::cout << "Forward: requesting frame " << f << " (" << pct << "%)\n"; + auto t8 = clock::now(); + timeline.Open(); + auto t9 = clock::now(); + std::cout << "Timeline Open(): " + << std::chrono::duration_cast(t9 - t8).count() + << " ms\n"; - cache->Seek(f); // signal “I need frame f now (and please prefetch f+1, f+2, …)” - std::shared_ptr framePtr = timeline.GetFrame(f); - writer.WriteFrame(framePtr); + for (int64_t frame_number : frames_to_fetch) { + auto loop_start = clock::now(); + std::cout << "Requesting frame " << frame_number << "...\n"; + + auto t10 = clock::now(); + std::shared_ptr frame = timeline.GetFrame(frame_number); + auto t11 = clock::now(); + std::cout << "Timeline GetFrame(" << frame_number << "): " + << std::chrono::duration_cast(t11 - t10).count() + << " ms\n"; + + std::string out_path = output_dir + "/frame-" + base + "-" + std::to_string(frame_number) + ".jpg"; + + auto t12 = clock::now(); + frame->Thumbnail(out_path, 200, 80, "", "", "#000000", false, "JPEG", 95, 0.0f); + auto t13 = clock::now(); + std::cout << "Frame Thumbnail() JPEG (" << frame_number << "): " + << std::chrono::duration_cast(t13 - t12).count() + << " ms\n"; + + auto loop_end = clock::now(); + std::cout << "Frame loop total (" << frame_number << "): " + << std::chrono::duration_cast(loop_end - loop_start).count() + << " ms\n"; + } + + reader.Close(); + timeline.Close(); + + auto file_end = clock::now(); + std::cout << "File total (" << base << "): " + << std::chrono::duration_cast(file_end - file_start).count() + << " ms\n"; } - auto t1 = std::chrono::high_resolution_clock::now(); - auto forward_ms = std::chrono::duration_cast(t1 - t0).count(); - // 5) Backward pass: same idea in reverse - auto t2 = std::chrono::high_resolution_clock::now(); - cache->setSpeed(-1); - for (int64_t f = total_frames; f >= 1; --f) { - float pct = (float(total_frames - f + 1) / total_frames) * 100.0f; - std::cout << "Backward: requesting frame " << f << " (" << pct << "%)\n"; - - cache->Seek(f); - std::shared_ptr framePtr = timeline.GetFrame(f); - writer.WriteFrame(framePtr); - } - auto t3 = std::chrono::high_resolution_clock::now(); - auto backward_ms = std::chrono::duration_cast(t3 - t2).count(); - - std::cout << "\nForward pass elapsed: " << forward_ms << " ms\n"; - std::cout << "Backward pass elapsed: " << backward_ms << " ms\n"; - - // 6) Shut down the cache thread, close everything - cache->StopThread(10000); // politely tells run() to exit, waits up to 10s - reader.Close(); - writer.Close(); - timeline.Close(); + auto total_end = clock::now(); + std::cout << "Total elapsed: " + << std::chrono::duration_cast(total_end - total_start).count() + << " ms\n"; return 0; } diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 82c595f3..e0d1b5bc 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -79,7 +79,9 @@ FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader) FFmpegReader::FFmpegReader(const std::string &path, DurationStrategy duration_strategy, bool inspect_reader) : last_frame(0), is_seeking(0), seeking_pts(0), seeking_frame(0), seek_count(0), NO_PTS_OFFSET(-99999), path(path), is_video_seek(true), check_interlace(false), check_fps(false), enable_seek(true), is_open(false), - seek_audio_frame_found(0), seek_video_frame_found(0),is_duration_known(false), largest_frame_processed(0), + seek_audio_frame_found(0), seek_video_frame_found(0), + last_seek_max_frame(-1), seek_stagnant_count(0), + is_duration_known(false), largest_frame_processed(0), current_video_frame(0), packet(NULL), duration_strategy(duration_strategy), audio_pts(0), video_pts(0), pFormatCtx(NULL), videoStream(-1), audioStream(-1), pCodecCtx(NULL), aCodecCtx(NULL), pStream(NULL), aStream(NULL), pFrame(NULL), previous_packet_location{-1,0}, @@ -136,52 +138,69 @@ static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPi { const enum AVPixelFormat *p; + // Prefer only the format matching the selected hardware decoder + int selected = openshot::Settings::Instance()->HARDWARE_DECODER; + for (p = pix_fmts; *p != AV_PIX_FMT_NONE; p++) { switch (*p) { #if defined(__linux__) // Linux pix formats case AV_PIX_FMT_VAAPI: - hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI; - return *p; + if (selected == 1) { + hw_de_av_pix_fmt_global = AV_PIX_FMT_VAAPI; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VAAPI; + return *p; + } break; case AV_PIX_FMT_VDPAU: - hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU; - return *p; + if (selected == 6) { + hw_de_av_pix_fmt_global = AV_PIX_FMT_VDPAU; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VDPAU; + return *p; + } break; #endif #if defined(_WIN32) // Windows pix formats case AV_PIX_FMT_DXVA2_VLD: - hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2; - return *p; + if (selected == 3) { + hw_de_av_pix_fmt_global = AV_PIX_FMT_DXVA2_VLD; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_DXVA2; + return *p; + } break; case AV_PIX_FMT_D3D11: - hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA; - return *p; + if (selected == 4) { + hw_de_av_pix_fmt_global = AV_PIX_FMT_D3D11; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_D3D11VA; + return *p; + } break; #endif #if defined(__APPLE__) // Apple pix formats case AV_PIX_FMT_VIDEOTOOLBOX: - hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; - return *p; + if (selected == 5) { + hw_de_av_pix_fmt_global = AV_PIX_FMT_VIDEOTOOLBOX; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; + return *p; + } break; #endif // Cross-platform pix formats case AV_PIX_FMT_CUDA: - hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA; - return *p; + if (selected == 2) { + hw_de_av_pix_fmt_global = AV_PIX_FMT_CUDA; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_CUDA; + return *p; + } break; case AV_PIX_FMT_QSV: - hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV; - hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV; - return *p; + if (selected == 7) { + hw_de_av_pix_fmt_global = AV_PIX_FMT_QSV; + hw_de_av_device_type_global = AV_HWDEVICE_TYPE_QSV; + return *p; + } break; default: // This is only here to silence unused-enum warnings @@ -302,7 +321,7 @@ void FFmpegReader::Open() { char *adapter_ptr = NULL; int adapter_num; adapter_num = openshot::Settings::Instance()->HW_DE_DEVICE_SET; - fprintf(stderr, "Hardware decoding device number: %d\n", adapter_num); + ZmqLogger::Instance()->AppendDebugMethod("Hardware decoding device number", "adapter_num", adapter_num); // Set hardware pix format (callback) pCodecCtx->get_format = get_hw_dec_format; @@ -388,6 +407,10 @@ void FFmpegReader::Open() { hw_device_ctx = NULL; // Here the first hardware initialisations are made if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { + const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type); + std::string hw_msg = "HW decode active: "; + hw_msg += (hw_name ? hw_name : "unknown"); + ZmqLogger::Instance()->Log(hw_msg); if (!(pCodecCtx->hw_device_ctx = av_buffer_ref(hw_device_ctx))) { throw InvalidCodec("Hardware device reference create failed.", path); } @@ -420,7 +443,8 @@ void FFmpegReader::Open() { */ } else { - throw InvalidCodec("Hardware device create failed.", path); + ZmqLogger::Instance()->Log("HW decode active: no (falling back to software)"); + throw InvalidCodec("Hardware device create failed.", path); } } #endif // USE_HW_ACCEL @@ -1035,6 +1059,8 @@ bool FFmpegReader::GetIsDurationKnown() { } std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { + last_seek_max_frame = -1; + seek_stagnant_count = 0; // Check for open reader (or throw exception) if (!is_open) throw ReaderClosed("The FFmpegReader is closed. Call Open() before calling this method.", path); @@ -1137,7 +1163,7 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { // Check the status of a seek (if any) if (is_seeking) { - check_seek = CheckSeek(false); + check_seek = CheckSeek(); } else { check_seek = false; } @@ -1446,9 +1472,12 @@ bool FFmpegReader::GetAVFrame() { } // Check the current seek position and determine if we need to seek again -bool FFmpegReader::CheckSeek(bool is_video) { +bool FFmpegReader::CheckSeek() { // Are we seeking for a specific frame? if (is_seeking) { + const int64_t kSeekRetryMax = 5; + const int kSeekStagnantMax = 2; + // Determine if both an audio and video packet have been decoded since the seek happened. // If not, allow the ReadStream method to keep looping if ((is_video_seek && !seek_video_frame_found) || (!is_video_seek && !seek_audio_frame_found)) @@ -1460,6 +1489,13 @@ bool FFmpegReader::CheckSeek(bool is_video) { // Determine max seeked frame int64_t max_seeked_frame = std::max(seek_audio_frame_found, seek_video_frame_found); + // Track stagnant seek results (no progress between retries) + if (max_seeked_frame == last_seek_max_frame) { + seek_stagnant_count++; + } else { + last_seek_max_frame = max_seeked_frame; + seek_stagnant_count = 0; + } // determine if we are "before" the requested frame if (max_seeked_frame >= seeking_frame) { @@ -1473,7 +1509,22 @@ bool FFmpegReader::CheckSeek(bool is_video) { "seek_audio_frame_found", seek_audio_frame_found); // Seek again... to the nearest Keyframe - Seek(seeking_frame - (10 * seek_count * seek_count)); + if (seek_count < kSeekRetryMax) { + Seek(seeking_frame - (10 * seek_count * seek_count)); + } else { + if (seek_stagnant_count >= kSeekStagnantMax) { + // Overshot and no progress: restart from the beginning and walk forward + Seek(1); + is_seeking = false; + seeking_frame = 0; + seeking_pts = -1; + } else { + // Give up retrying and walk forward + is_seeking = false; + seeking_frame = 0; + seeking_pts = -1; + } + } } else { // SEEK WORKED ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckSeek (Successful)", @@ -1980,7 +2031,7 @@ void FFmpegReader::Seek(int64_t requested_frame) { if (!seek_worked && info.has_video && !HasAlbumArt()) { seek_target = ConvertFrameToVideoPTS(requested_frame - buffer_amount); if (av_seek_frame(pFormatCtx, info.video_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) { - fprintf(stderr, "%s: error while seeking video stream\n", pFormatCtx->AV_FILENAME); + ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking video stream"); } else { // VIDEO SEEK is_video_seek = true; @@ -1992,7 +2043,7 @@ void FFmpegReader::Seek(int64_t requested_frame) { if (!seek_worked && info.has_audio) { seek_target = ConvertFrameToAudioPTS(requested_frame - buffer_amount); if (av_seek_frame(pFormatCtx, info.audio_stream_index, seek_target, AVSEEK_FLAG_BACKWARD) < 0) { - fprintf(stderr, "%s: error while seeking audio stream\n", pFormatCtx->AV_FILENAME); + ZmqLogger::Instance()->Log(std::string(pFormatCtx->AV_FILENAME) + ": error while seeking audio stream"); } else { // AUDIO SEEK is_video_seek = false; diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index f264754c..23ce70e0 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -136,6 +136,8 @@ namespace openshot { int seek_count; int64_t seek_audio_frame_found; int64_t seek_video_frame_found; + int64_t last_seek_max_frame; + int seek_stagnant_count; int64_t last_frame; int64_t largest_frame_processed; @@ -172,7 +174,7 @@ namespace openshot { void CheckFPS(); /// Check the current seek position and determine if we need to seek again - bool CheckSeek(bool is_video); + bool CheckSeek(); /// Check the working queue, and move finished frames to the finished queue void CheckWorkingFrames(int64_t requested_frame);