diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d11cc72b..5f461d9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ linux-builder: - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 ../ - make -j 4 - make install + - ctest --output-on-failure -VV - make doc - ~/auto-update-docs "$CI_PROJECT_DIR/build" "$CI_COMMIT_REF_NAME" - PROJECT_VERSION=$(grep -E '^set\(PROJECT_VERSION_FULL "(.*)' ../CMakeLists.txt | awk '{print $2}' | tr -d '")') @@ -66,7 +67,7 @@ mac-builder: - cmake -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.12" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ - make -j 9 - make install - - make test + - ctest --output-on-failure -VV - PROJECT_VERSION=$(grep -E '^set\(PROJECT_VERSION_FULL "(.*)' ../CMakeLists.txt | awk '{print $2}' | tr -d '")') - PROJECT_SO=$(grep -E '^set\(PROJECT_SO_VERSION (.*)' ../CMakeLists.txt | awk '{print $2}' | tr -d ')') - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$PROJECT_VERSION\nSO:$PROJECT_SO" > "install-x64/share/$CI_PROJECT_NAME.env" @@ -91,7 +92,7 @@ windows-builder-x64: - $env:MSYSTEM = "MINGW64" - cmake -B build -S . -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" - cmake --build build -j 4 - - cmake --build build --target coverage + - ctest --test-dir build --output-on-failure -VV - cmake --install build - $PROJECT_VERSION = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_VERSION_FULL "(.*)\"' | %{$_.Matches.Groups[1].value}) - $PROJECT_SO = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_SO_VERSION (.*)\)' | %{$_.Matches.Groups[1].value}) diff --git a/doc/HW-ACCEL.md b/doc/HW-ACCEL.md index e01513cc..4960777b 100644 --- a/doc/HW-ACCEL.md +++ b/doc/HW-ACCEL.md @@ -6,55 +6,187 @@ SPDX-License-Identifier: LGPL-3.0-or-later ## Hardware Acceleration -OpenShot now has experimental support for hardware acceleration, which uses 1 (or more) -graphics cards to offload some of the work for both decoding and encoding. This is -very new and experimental (as of May 2019), but we look forward to "accelerating" -our support for this in the future! +Hardware acceleration in libopenshot allows FFmpeg to use platform-specific GPU +APIs for video decode and encode when available. In practice, this means some of +the work that would otherwise be done by the CPU can be offloaded to the GPU or +to dedicated media blocks on the GPU. -The following table summarizes our current level of support: +This document focuses on what hardware acceleration in libopenshot does today, +how it fits into the current processing pipeline, and what users and developers +should expect from it. -| | Linux Decode | Linux Encode | Mac Decode | Mac Encode | Windows Decode | Windows Encode | Notes | -|--------------------|:---------------:|:--------------:|:----------:|:--------------:|:--------------:|:--------------:|------------------| -| VA-API | ✔️   | ✔️   | - | - | - | - | *Linux Only* | -| VDPAU | ✔️ 1 | ✅ 2 | - | - | - | - | *Linux Only* | -| CUDA (NVDEC/NVENC) | ❌ 3 | ✔️   | - | - | - | ✔️   | *Cross Platform* | -| VideoToolBox | - | - | ✔️   | ❌ 4 | - | - | *Mac Only* | -| DXVA2 | - | - | - | - | ❌ 3 | - | *Windows Only* | -| D3D11VA | - | - | - | - | ❌ 3 | - | *Windows Only* | -| QSV | ❌ 3 | ❌   | ❌   | ❌   | ❌   | ❌   | *Cross Platform* | +## Backend Overview + +The following table summarizes the historically supported hardware-acceleration +backends in libopenshot. Actual behavior still depends on FFmpeg build options, +driver availability, operating system support, and the runtime environment. + +| | Linux Decode | Linux Encode | macOS Decode | macOS Encode | Windows Decode | Windows Encode | Notes | +|--------------------|:------------:|:------------:|:------------:|:------------:|:--------------:|:--------------:|-------| +| VA-API | ✔️ | ✔️ | - | - | - | - | Linux only | +| VDPAU | ✔️ 1 | ✅ 2 | - | - | - | - | Linux only | +| CUDA (NVDEC/NVENC) | ❌ 3 | ✔️ | - | - | - | ✔️ | Backend availability depends on the FFmpeg build | +| VideoToolbox | - | - | ✔️ | ❌ 4 | - | - | macOS only | +| DXVA2 | - | - | - | - | ❌ 3 | - | Windows only | +| D3D11VA | - | - | - | - | ❌ 3 | - | Windows only | +| QSV | ❌ 3 | ❌ | ❌ | ❌ | ❌ | ❌ | Backend availability depends on the FFmpeg build | #### Notes -1. VDPAU for some reason needs a card number one higher than it really is -2. VDPAU is a decoder only -3. Green frames (pixel data not correctly tranferred back to system memory) -4. Crashes and burns +1. VDPAU historically needed a card number one higher than expected. +2. VDPAU is decode-only. +3. Historically associated with failed transfers, corrupt frames, or unusable output on some setups. +4. Historically unstable. + +This table should be read as a support map, not a guarantee that every backend +is fully validated on every current OS/driver combination. + +## Why Hardware Acceleration Exists + +Hardware acceleration is useful for two main reasons: + +* It can reduce CPU load during decode and encode. +* It can improve throughput for some media, especially on systems with strong + hardware video support. + +However, hardware acceleration is not automatically faster for every file or on +every system. The real result depends on codec support, driver quality, stream +format, pixel format, resolution, frame rate, and how much CPU-side work still +needs to happen after decode. + +## What libopenshot Uses Hardware Acceleration For + +Today, hardware acceleration in libopenshot is primarily used for: + +* video decode +* video encode + +It is not currently used to keep the entire edit/render pipeline on the GPU. +Decoded frames usually still need to be copied back into system memory for +colorspace conversion, scaling, caching, effect processing, compositing, and +timeline rendering. + +That detail is important because it explains why hardware decode does not always +produce a speedup. + +## Decode Flow in libopenshot + +The current decode flow looks roughly like this: + +1. A hardware decoder may be requested through `Settings::HARDWARE_DECODER`. +2. FFmpeg opens the requested hardware decode path if the backend and driver + support it. +3. The decoder produces a frame, either: + * directly as a software-readable frame, or + * as a hardware frame that must be transferred to system memory. +4. libopenshot converts that frame into the CPU-side image representation used + by the rest of the pipeline. + +If hardware decode fails during startup decode or frame transfer, libopenshot +falls back to software decode for that reader instead of returning corrupt, +green, or black frames. + +## Fallback Behavior + +Hardware decode is best-effort, not all-or-nothing. + +If a hardware decoder is requested and one of the following happens early in the +decode path: + +* repeated startup decode failures +* failed hardware-frame transfer +* invalid transferred frame data +* failed software conversion of a transferred frame + +libopenshot reopens that reader in software decode mode and continues decoding. + +This behavior is intentionally conservative. The priority is correctness and +stability: + +* valid frames are better than corrupt frames +* software fallback is better than black or green output +* a file that cannot be decoded by one hardware backend should still decode if + CPU decoding can handle it + +For diagnostics and UI checks, this means there is a difference between: + +* decode succeeded +* hardware decode actually produced frames +* hardware decode failed and software fallback was used + +`FFmpegReader::HardwareDecodeSuccessful()` exists to expose that distinction. + +## Performance Expectations + +Hardware decode is not guaranteed to be faster than software decode. + +In libopenshot's current pipeline, decoded frames are brought back to +system memory immediately after decode. That introduces costs that can erase or +outweigh the raw decode benefit: + +* hardware device setup overhead +* frame transfer overhead between GPU and CPU memory +* colorspace conversion and scaling after decode +* caching and image wrapping in CPU memory +* container/seek behavior and stream structure + +Because of that, hardware decode performance is workload-dependent. + +General guidance: + +* some files benefit from hardware decode +* some files are effectively neutral +* some files are slower with hardware decode +* files with similar codec and resolution can still behave differently + +Hardware acceleration should be treated as a capability that may help, not as a +guarantee of better performance. + +## Why Some Files Fail on Hardware Decode + +A file can fail on hardware decode for several reasons: + +* unsupported codec profile +* unsupported chroma format or pixel format +* unsupported bit depth or color range +* driver/backend limitations +* FFmpeg/backend integration quirks on a specific platform + +For example, consumer hardware decode paths often handle H.264 4:2:0 very well, +but may not support H.264 4:2:2 decode reliably. In those cases, software decode +may work perfectly while hardware decode fails. ## Supported FFmpeg Versions -* HW accel is supported from FFmpeg version 3.4 -* HW accel was removed for nVidia drivers in Ubuntu for FFmpeg 4+ +* Hardware acceleration support requires FFmpeg versions new enough to expose the + relevant hardware APIs to libopenshot. +* In practice, decode support in libopenshot relies on FFmpeg's modern send/receive + decode API and hardware-frame APIs. +* Actual backend availability depends on how FFmpeg was compiled on the target system. -**Notice:** The FFmpeg versions of Ubuntu and PPAs for Ubuntu show the -same behaviour. FFmpeg 3 has working nVidia hardware acceleration while -FFmpeg 4+ has no support for nVidia hardware acceleration -included. +Older historical note: + +* Some Ubuntu/FFmpeg/NVIDIA combinations behaved differently between FFmpeg 3.x + and FFmpeg 4.x, especially for NVIDIA decode support. + +Because backend support has changed over time, always validate against the +actual FFmpeg build and driver stack in use. ## OpenShot Settings -The following settings are use by libopenshot to enable, disable, and control -the various hardware acceleration features. +The following settings are used by libopenshot to enable, disable, and control +hardware acceleration features. -```{cpp} +```cpp /// Use video codec for faster video decoding (if supported) int HARDWARE_DECODER = 0; /* 0 - No acceleration 1 - Linux VA-API - 2 - nVidia NVDEC + 2 - NVIDIA NVDEC 3 - Windows D3D9 4 - Windows D3D11 - 5 - MacOS / VideoToolBox + 5 - macOS / VideoToolbox 6 - Linux VDPAU 7 - Intel QSV */ @@ -70,73 +202,91 @@ int DE_LIMIT_HEIGHT_MAX = 1100; /// Maximum columns that hardware decode can handle int DE_LIMIT_WIDTH_MAX = 1950; -/// Which GPU to use to decode (0 is the first, LINUX ONLY) +/// Which GPU to use to decode (0 is the first, Linux only) int HW_DE_DEVICE_SET = 0; -/// Which GPU to use to encode (0 is the first, LINUX ONLY) +/// Which GPU to use to encode (0 is the first, Linux only) int HW_EN_DEVICE_SET = 0; ``` -## Libva / VA-API (Video Acceleration API) +## Platform Notes -The correct version of libva is needed (libva in Ubuntu 16.04 or libva2 -in Ubuntu 18.04) for the AppImage to work with hardware acceleration. -An AppImage that works on both systems (supporting libva and libva2), -might be possible when no libva is included in the AppImage. +### Linux / VA-API -* vaapi is working for intel and AMD -* vaapi is working for decode only for nouveau -* nVidia driver is working for export only +VA-API is one of the primary Linux hardware-decode paths used by libopenshot. +On supported Intel and AMD systems it can work well, but not every file format, +codec profile, or pixel format is supported by every driver. -## AMD Graphics Cards (RadeonOpenCompute/ROCm) +### Linux / VDPAU -Decoding and encoding on the (AMD) GPU is possible with the default drivers. -On systems where ROCm is installed and run a future use for GPU acceleration -of effects could be implemented (contributions welcome). +VDPAU support exists historically, but behavior can vary with driver and FFmpeg +stack. Treat it as backend-dependent rather than universally reliable. + +### NVIDIA + +NVIDIA hardware encode support has historically been more reliable than decode +support in libopenshot, depending on FFmpeg build and driver stack. Validate the +actual runtime environment before assuming support. + +### macOS / VideoToolbox + +VideoToolbox support exists, but stability and feature coverage should be tested +carefully on the target FFmpeg/macOS version. + +### Windows / DXVA2 / D3D11VA + +Windows decode backends are highly dependent on FFmpeg build options and device +support. They should be treated as runtime-validated features, not assumptions. ## Multiple Graphics Cards -If the computer has multiple graphics cards installed, you can choose which -should be used by libopenshot. Also, you can optionally use one card for -decoding and the other for encoding (if both cards support acceleration). -This is currently only supported on Linux, due to the device name FFmpeg -expects (i.e. **/dev/dri/render128**). Contributions welcome if anyone can -determine what string format to pass for Windows and Mac. +If the computer has multiple graphics cards installed, libopenshot can choose +which device should be used for decode and encode. This is currently practical +mainly on Linux, where FFmpeg expects device names such as `/dev/dri/renderD128`. -## Help Us Improve Hardware Support +Contributions are welcome for improving cross-platform device enumeration and +selection. -This information might be wrong, and we would love to continue improving -our support for hardware acceleration in OpenShot. Please help us update -this document if you find an error or discover new and/or useful information. +## Testing and Validation -**FFmpeg 4 + nVidia** The manual at: -https://www.tal.org/tutorials/ffmpeg_nvidia_encode -works pretty well. We could compile and install a version of FFmpeg 4.1.3 -on Mint 19.1 that supports the GPU on nVidia cards. A version of openshot -with hardware support using these libraries could use the nVidia GPU. +When validating hardware decode, check both: -**BUG:** Hardware supported decoding still has some bugs (as you can see from -the chart above). Also, the speed gains with decoding are not as great -as with encoding. Currently, if hardware decoding fails, there is no -fallback (you either get green frames or an "invalid file" error in OpenShot). -This needs to be improved to successfully fall-back to software decoding. +* correctness of the decoded output +* whether hardware decode actually succeeded -**Needed:** - * A way to get options and limits of the GPU, such as - supported dimensions (width and height). - * A way to list the actual Graphic Cards available to FFmpeg (for the - user to choose which card for decoding and encoding, as opposed - to "Graphics Card X") +A frame that looks correct is not enough to prove that hardware acceleration +worked, because software fallback may have rescued the decode. -**Further improvement:** Right now the frame can be decoded on the GPU, but the -frame is then copied to CPU memory for modifications. It is then copied back to -GPU memory for encoding. Using the GPU for both decoding and modifications -will make it possible to do away with these two copies. A possible solution would -be to use Vulkan compute which would be available on Linux and Windows natively -and on MacOS via MoltenVK. +Recommended validation: + +* compare output against a software-decode baseline +* track whether hardware decode actually produced frames +* test both a known-good hardware sample and a known-failing fallback sample + +## Future Improvements + +The biggest architectural limitation today is that decoded frames are generally +copied back to CPU memory for the rest of the pipeline. + +Longer-term improvements could include: + +* better hardware capability probing +* better device enumeration for users +* broader backend validation across platforms +* keeping more of the pipeline on GPU memory +* GPU-native effects/compositing paths + +Avoiding repeated GPU-to-CPU and CPU-to-GPU copies would make hardware +acceleration much more effective for end-to-end editing and export workflows. + +## Help Improve This Document + +Hardware acceleration support changes with FFmpeg, drivers, operating systems, +and GPU generations. If you find incorrect information or validate a backend on +a newer stack, please update this document. ## Credit -A big thanks to Peter M (https://github.com/eisneinechse) for all his work -on integrating hardware acceleration into libopenshot! The community thanks -you for this major contribution! +A big thanks to Peter M (https://github.com/eisneinechse) for all his work on +integrating hardware acceleration into libopenshot. The community thanks you for +this major contribution. diff --git a/examples/test_aac_preroll.mp4 b/examples/test_aac_preroll.mp4 new file mode 100644 index 00000000..d46ffb88 Binary files /dev/null and b/examples/test_aac_preroll.mp4 differ diff --git a/src/Clip.cpp b/src/Clip.cpp index 022c0065..59a2672f 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef USE_IMAGEMAGICK #include "MagickUtilities.h" @@ -512,18 +513,36 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun final_cache.Add(frame); } - if (!background_frame) { - // Create missing background_frame w/ transparent color (if needed) - background_frame = std::make_shared(frame->number, frame->GetWidth(), frame->GetHeight(), - "#00000000", frame->GetAudioSamplesCount(), - frame->GetAudioChannelsCount()); - } + const bool has_external_background = (background_frame != nullptr); - // Apply background canvas (i.e. flatten this image onto previous layer image) - apply_background(frame, background_frame); + // Timeline path. + if (options) { + if (!background_frame) { + // Create a transparent background if missing. + background_frame = std::make_shared(frame->number, frame->GetWidth(), frame->GetHeight(), + "#00000000", frame->GetAudioSamplesCount(), + frame->GetAudioChannelsCount()); + } + if (options->force_safe_composite) { + // Edit mode: composite without mutating cached frame pixels. + apply_background(frame, background_frame, false); + return frame; + } - // Return processed 'frame' - return frame; + // Playback mode: keep original fast path. + apply_background(frame, background_frame, true); + return frame; + } + + // No background: return the frame directly. + if (!has_external_background) { + return frame; + } + + // External background: composite on a copy. + auto output = std::make_shared(*frame.get()); + apply_background(output, background_frame, true); + return output; } else // Throw error if reader not initialized @@ -1263,7 +1282,9 @@ void Clip::RemoveEffect(EffectBase* effect) } // Apply background image to the current clip image (i.e. flatten this image onto previous layer) -void Clip::apply_background(std::shared_ptr frame, std::shared_ptr background_frame) { +void Clip::apply_background(std::shared_ptr frame, + std::shared_ptr background_frame, + bool update_frame_image) { // Add background canvas std::shared_ptr background_canvas = background_frame->GetImage(); QPainter painter(background_canvas.get()); @@ -1273,8 +1294,10 @@ void Clip::apply_background(std::shared_ptr frame, std::shared_ painter.drawImage(0, 0, *frame->GetImage()); painter.end(); - // Add new QImage to frame - frame->AddImage(background_canvas); + // Standalone clip requests update frame->image, but timeline composition + // draws onto the timeline-owned background frame only. + if (update_frame_image) + frame->AddImage(background_canvas); } // Apply effects to the source frame (if any) @@ -1284,9 +1307,9 @@ void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_nu { // Apply the effect to this frame if (effect->info.apply_before_clip && before_keyframes) { - effect->GetFrame(frame, frame->number); + effect->ProcessFrame(frame, frame->number); } else if (!effect->info.apply_before_clip && !before_keyframes) { - effect->GetFrame(frame, frame->number); + effect->ProcessFrame(frame, frame->number); } } diff --git a/src/Clip.h b/src/Clip.h index cfb37768..d0304462 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -128,7 +128,9 @@ namespace openshot { int64_t adjust_frame_number_minimum(int64_t frame_number); /// Apply background image to the current clip image (i.e. flatten this image onto previous layer) - void apply_background(std::shared_ptr frame, std::shared_ptr background_frame); + void apply_background(std::shared_ptr frame, + std::shared_ptr background_frame, + bool update_frame_image = true); /// Apply effects to the source frame (if any) void apply_effects(std::shared_ptr frame, int64_t timeline_frame_number, TimelineInfoStruct* options, bool before_keyframes); diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index d833b8cb..5bc8a0fd 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -12,11 +12,24 @@ #include #include +#include +#include #include "EffectBase.h" #include "Exceptions.h" +#include "Clip.h" #include "Timeline.h" +#include "ReaderBase.h" +#include "ChunkReader.h" +#include "FFmpegReader.h" +#include "QtImageReader.h" +#include "ZmqLogger.h" +#include + +#ifdef USE_IMAGEMAGICK + #include "ImageReader.h" +#endif using namespace openshot; @@ -31,6 +44,10 @@ void EffectBase::InitEffectInfo() Order(0); ParentClip(NULL); parentEffect = NULL; + mask_invert = false; + mask_reader = NULL; + mask_time_mode = MASK_TIME_SOURCE_FPS; + mask_loop_mode = MASK_LOOP_PLAY_ONCE; info.has_video = false; info.has_audio = false; @@ -89,6 +106,13 @@ Json::Value EffectBase::JsonValue() const { root["has_tracked_object"] = info.has_tracked_object; root["apply_before_clip"] = info.apply_before_clip; root["order"] = Order(); + root["mask_invert"] = mask_invert; + root["mask_time_mode"] = mask_time_mode; + root["mask_loop_mode"] = mask_loop_mode; + if (mask_reader) + root["mask_reader"] = mask_reader->JsonValue(); + else + root["mask_reader"] = Json::objectValue; // return JsonValue return root; @@ -141,6 +165,13 @@ void EffectBase::SetJsonValue(const Json::Value root) { my_root = root; } + // Legacy compatibility: older shared-mask JSON stored source trim + // separately from the effect trim. Canonical trim now uses ClipBase. + if (my_root["start"].isNull() && !my_root["mask_start"].isNull()) + my_root["start"] = my_root["mask_start"]; + if (my_root["end"].isNull() && !my_root["mask_end"].isNull()) + my_root["end"] = my_root["mask_end"]; + // Set parent data ClipBase::SetJsonValue(my_root); @@ -151,6 +182,32 @@ void EffectBase::SetJsonValue(const Json::Value root) { if (!my_root["apply_before_clip"].isNull()) info.apply_before_clip = my_root["apply_before_clip"].asBool(); + if (!my_root["mask_invert"].isNull()) + mask_invert = my_root["mask_invert"].asBool(); + if (!my_root["mask_time_mode"].isNull()) { + const int time_mode = my_root["mask_time_mode"].asInt(); + mask_time_mode = (time_mode == MASK_TIME_TIMELINE || time_mode == MASK_TIME_SOURCE_FPS) + ? time_mode : MASK_TIME_SOURCE_FPS; + } + if (!my_root["mask_loop_mode"].isNull()) { + const int loop_mode = my_root["mask_loop_mode"].asInt(); + if (loop_mode >= MASK_LOOP_PLAY_ONCE && loop_mode <= MASK_LOOP_PING_PONG) + mask_loop_mode = loop_mode; + else + mask_loop_mode = MASK_LOOP_PLAY_ONCE; + } + + const Json::Value mask_reader_json = + !my_root["mask_reader"].isNull() ? my_root["mask_reader"] : my_root["reader"]; + + if (!mask_reader_json.isNull()) { + if (!mask_reader_json["type"].isNull()) { + MaskReader(CreateReaderFromJson(mask_reader_json)); + } else if (mask_reader_json.isObject() && mask_reader_json.empty()) { + MaskReader(NULL); + } + } + if (!my_root["parent_effect_id"].isNull()){ info.parent_effect_id = my_root["parent_effect_id"].asString(); if (info.parent_effect_id.size() > 0 && info.parent_effect_id != "" && parentEffect == NULL) @@ -194,9 +251,300 @@ Json::Value EffectBase::BasePropertiesJSON(int64_t requested_frame) const { // Set the parent effect which properties this effect will inherit root["parent_effect_id"] = add_property_json("Parent", 0.0, "string", info.parent_effect_id, NULL, -1, -1, false, requested_frame); + if (info.has_video) { + root["mask_invert"] = add_property_json("Mask: Invert", mask_invert, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_invert"]["choices"].append(add_property_choice_json("Yes", true, mask_invert)); + root["mask_invert"]["choices"].append(add_property_choice_json("No", false, mask_invert)); + + root["mask_time_mode"] = add_property_json("Mask: Time Mode", mask_time_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_time_mode"]["choices"].append(add_property_choice_json("Timeline", MASK_TIME_TIMELINE, mask_time_mode)); + root["mask_time_mode"]["choices"].append(add_property_choice_json("Source FPS", MASK_TIME_SOURCE_FPS, mask_time_mode)); + + root["mask_loop_mode"] = add_property_json("Mask: Loop", mask_loop_mode, "int", "", NULL, 0, 2, false, requested_frame); + root["mask_loop_mode"]["choices"].append(add_property_choice_json("Play Once", MASK_LOOP_PLAY_ONCE, mask_loop_mode)); + root["mask_loop_mode"]["choices"].append(add_property_choice_json("Repeat", MASK_LOOP_REPEAT, mask_loop_mode)); + root["mask_loop_mode"]["choices"].append(add_property_choice_json("Ping-Pong", MASK_LOOP_PING_PONG, mask_loop_mode)); + + if (mask_reader) + root["mask_reader"] = add_property_json("Mask: Source", 0.0, "reader", mask_reader->Json(), NULL, 0, 1, false, requested_frame); + else + root["mask_reader"] = add_property_json("Mask: Source", 0.0, "reader", "{}", NULL, 0, 1, false, requested_frame); + } + return root; } +ReaderBase* EffectBase::CreateReaderFromJson(const Json::Value& reader_json) const { + if (reader_json["type"].isNull()) + return NULL; + + ReaderBase* reader = NULL; + const std::string type = reader_json["type"].asString(); + + if (type == "FFmpegReader") { + reader = new FFmpegReader(reader_json["path"].asString()); + reader->SetJsonValue(reader_json); + // Mask readers are video-only sources. Disabling audio avoids FFmpeg + // A/V readiness fallbacks that can repeat stale video frames. + reader->info.has_audio = false; + reader->info.audio_stream_index = -1; + } else if (type == "QtImageReader") { + reader = new QtImageReader(reader_json["path"].asString()); + reader->SetJsonValue(reader_json); + } else if (type == "ChunkReader") { + reader = new ChunkReader(reader_json["path"].asString(), + static_cast(reader_json["chunk_version"].asInt())); + reader->SetJsonValue(reader_json); +#ifdef USE_IMAGEMAGICK + } else if (type == "ImageReader") { + reader = new ImageReader(reader_json["path"].asString()); + reader->SetJsonValue(reader_json); +#endif + } + + return reader; +} + +void EffectBase::MaskReader(ReaderBase* new_reader) { + if (mask_reader == new_reader) + return; + + if (mask_reader) { + mask_reader->Close(); + delete mask_reader; + } + + mask_reader = new_reader; + cached_single_mask_image.reset(); + cached_single_mask_width = 0; + cached_single_mask_height = 0; + if (mask_reader) + mask_reader->ParentClip(clip); +} + +double EffectBase::ResolveMaskHostFps() { + if (clip) { + Clip* parent_clip = dynamic_cast(clip); + if (parent_clip && parent_clip->info.fps.num > 0 && parent_clip->info.fps.den > 0) + return parent_clip->info.fps.ToDouble(); + } + + Timeline* parent_timeline = dynamic_cast(ParentTimeline()); + if (parent_timeline && parent_timeline->info.fps.num > 0 && parent_timeline->info.fps.den > 0) + return parent_timeline->info.fps.ToDouble(); + + if (mask_reader && mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) + return mask_reader->info.fps.ToDouble(); + + return 30.0; +} + +double EffectBase::ResolveMaskSourceDuration() const { + if (!mask_reader) + return 0.0; + + if (mask_reader->info.duration > 0.0f) + return static_cast(mask_reader->info.duration); + + if (mask_reader->info.video_length > 0 && + mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) { + return static_cast(mask_reader->info.video_length) / mask_reader->info.fps.ToDouble(); + } + + return 0.0; +} + +int64_t EffectBase::MapMaskFrameNumber(int64_t frame_number) { + if (!mask_reader) + return frame_number; + + int64_t requested_index = std::max(int64_t(0), frame_number - 1); + if (!clip && ParentTimeline()) { + const double host_fps = ResolveMaskHostFps(); + if (host_fps > 0.0) { + const int64_t start_offset = static_cast(std::llround(std::max(0.0f, Start()) * host_fps)); + requested_index = std::max(int64_t(0), requested_index - start_offset); + } + } + int64_t mapped_index = requested_index; + + if (mask_time_mode == MASK_TIME_SOURCE_FPS && + mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) { + const double host_fps = ResolveMaskHostFps(); + const double source_fps = mask_reader->info.fps.ToDouble(); + if (host_fps > 0.0 && source_fps > 0.0) { + const double seconds = static_cast(requested_index) / host_fps; + mapped_index = static_cast(std::llround(seconds * source_fps)); + } + } + + const int64_t source_len = mask_reader->info.video_length; + const double source_fps = (mask_reader->info.fps.num > 0 && mask_reader->info.fps.den > 0) + ? mask_reader->info.fps.ToDouble() : 30.0; + const double source_duration = ResolveMaskSourceDuration(); + const double start_sec = std::min(std::max(0.0f, Start()), source_duration); + const double end_sec = std::min(std::max(0.0f, End()), source_duration); + + const int64_t range_start = std::max(int64_t(1), static_cast(std::llround(start_sec * source_fps)) + 1); + int64_t range_end = (end_sec > 0.0) + ? static_cast(std::llround(end_sec * source_fps)) + 1 + : source_len; + if (source_len > 0) + range_end = std::min(range_end, source_len); + if (range_end < range_start) + range_end = range_start; + + const int64_t range_len = std::max(int64_t(1), range_end - range_start + 1); + int64_t range_index = mapped_index; + + switch (mask_loop_mode) { + case MASK_LOOP_REPEAT: + range_index = mapped_index % range_len; + break; + case MASK_LOOP_PING_PONG: + if (range_len > 1) { + const int64_t cycle_len = (range_len * 2) - 2; + int64_t phase = mapped_index % cycle_len; + if (phase >= range_len) + phase = cycle_len - phase; + range_index = phase; + } else { + range_index = 0; + } + break; + case MASK_LOOP_PLAY_ONCE: + default: + if (mapped_index < 0) + range_index = 0; + else if (mapped_index >= range_len) + range_index = range_len - 1; + else + range_index = mapped_index; + break; + } + + int64_t mapped_frame = range_start + range_index; + if (source_len > 0) + mapped_frame = std::min(std::max(int64_t(1), mapped_frame), source_len); + return std::max(int64_t(1), mapped_frame); +} + +std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_image, int64_t frame_number) { + if (!mask_reader || !target_image || target_image->isNull()) + return {}; + + std::shared_ptr source_mask; + bool used_cached_scaled = false; + #pragma omp critical (open_effect_mask_reader) + { + try { + if (!mask_reader->IsOpen()) + mask_reader->Open(); + + if (mask_reader->info.has_single_image && + cached_single_mask_image && + cached_single_mask_width == target_image->width() && + cached_single_mask_height == target_image->height()) { + source_mask = cached_single_mask_image; + used_cached_scaled = true; + } + else { + const int64_t mapped_frame = MapMaskFrameNumber(frame_number); + auto source_frame = mask_reader->GetFrame(mapped_frame); + if (source_frame && source_frame->GetImage() && !source_frame->GetImage()->isNull()) + source_mask = std::make_shared(*source_frame->GetImage()); + } + } catch (const std::exception& e) { + ZmqLogger::Instance()->Log( + std::string("EffectBase::GetMaskImage unable to read mask frame: ") + e.what()); + source_mask.reset(); + } + } + + if (!source_mask || source_mask->isNull()) + return {}; + + if (used_cached_scaled) + return source_mask; + + auto scaled_mask = std::make_shared( + source_mask->scaled( + target_image->width(), target_image->height(), + Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); + if (mask_reader->info.has_single_image) { + cached_single_mask_image = scaled_mask; + cached_single_mask_width = target_image->width(); + cached_single_mask_height = target_image->height(); + } + return scaled_mask; +} + +void EffectBase::BlendWithMask(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image) const { + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + int gray = qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2]); + if (mask_invert) + gray = 255 - gray; + const float factor = static_cast(gray) / 255.0f; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = static_cast( + (original_pixels[idx + 3] * inverse) + (effected_pixels[idx + 3] * factor)); + } +} + +std::shared_ptr EffectBase::ProcessFrame(std::shared_ptr frame, int64_t frame_number) { + // Audio-only effects skip common mask handling. + if (!info.has_video || !mask_reader) + return GetFrame(frame, frame_number); + + // Effects that already apply masks inside GetFrame() should bypass common blend handling. + if (HandlesMaskInternally()) + return GetFrame(frame, frame_number); + + auto pre_image = frame->GetImage(); + if (!pre_image || pre_image->isNull()) + return GetFrame(frame, frame_number); + + const auto original_image = std::make_shared(pre_image->copy()); + auto output_frame = GetFrame(frame, frame_number); + if (!output_frame) + return output_frame; + auto effected_image = output_frame->GetImage(); + if (!effected_image || effected_image->isNull() || + effected_image->size() != original_image->size()) + return output_frame; + + auto mask_image = GetMaskImage(effected_image, frame_number); + if (!mask_image || mask_image->isNull()) + return output_frame; + + if (UseCustomMaskBlend(frame_number)) + ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number); + else + BlendWithMask(original_image, effected_image, mask_image); + + return output_frame; +} + /// Parent clip object of this reader (which can be unparented and NULL) openshot::ClipBase* EffectBase::ParentClip() { return clip; @@ -205,6 +553,8 @@ openshot::ClipBase* EffectBase::ParentClip() { /// Set parent clip object of this reader void EffectBase::ParentClip(openshot::ClipBase* new_clip) { clip = new_clip; + if (mask_reader) + mask_reader->ParentClip(new_clip); } // Set the parent effect from which this properties will be set to @@ -239,3 +589,7 @@ std::string EffectBase::ParentClipId() const{ else return ""; } + +EffectBase::~EffectBase() { + MaskReader(NULL); +} diff --git a/src/EffectBase.h b/src/EffectBase.h index fb39d62b..8fa21888 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -18,12 +18,15 @@ #include "Json.h" #include "TrackedObjectBase.h" +#include #include #include #include namespace openshot { + class ReaderBase; + /** * @brief This struct contains info about an effect, such as the name, video or audio effect, etc... * @@ -54,10 +57,48 @@ namespace openshot { private: int order; ///< The order to evaluate this effect. Effects are processed in this order (when more than one overlap). + ReaderBase* mask_reader = nullptr; ///< Optional common reader-based mask source. + std::shared_ptr cached_single_mask_image; ///< Cached scaled mask for still-image mask sources. + int cached_single_mask_width = 0; ///< Cached mask width. + int cached_single_mask_height = 0; ///< Cached mask height. + + /// Build or refresh a mask image that matches target_image dimensions. + std::shared_ptr GetMaskImage(std::shared_ptr target_image, int64_t frame_number); + + /// Blend original and effected images using mask values. + void BlendWithMask(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image) const; protected: openshot::ClipBase* clip; ///< Pointer to the parent clip instance (if any) + /// Create a reader instance from reader JSON. + ReaderBase* CreateReaderFromJson(const Json::Value& reader_json) const; + + /// Convert an effect frame number to a mask source frame number. + int64_t MapMaskFrameNumber(int64_t frame_number); + + /// Determine host FPS used to convert timeline frames to mask source FPS. + double ResolveMaskHostFps(); + + /// Determine mask source duration in seconds. + double ResolveMaskSourceDuration() const; + + /// Resolve a cached/scaled mask image for the target frame dimensions. + std::shared_ptr ResolveMaskImage(std::shared_ptr target_image, int64_t frame_number) { + return GetMaskImage(target_image, frame_number); + } + + /// Optional override for effects that need custom mask behavior. + virtual bool UseCustomMaskBlend(int64_t frame_number) const { return false; } + + /// Optional override for effects with custom mask implementation. + virtual void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const {} + + /// Optional override for effects that apply mask processing inside GetFrame(). + virtual bool HandlesMaskInternally() const { return false; } + public: /// Parent effect (which properties will set this effect properties) EffectBase* parentEffect; @@ -67,6 +108,21 @@ namespace openshot /// Information about the current effect EffectInfoStruct info; + bool mask_invert = false; ///< Invert grayscale mask values before blending. + + enum MaskTimeMode { + MASK_TIME_TIMELINE = 0, + MASK_TIME_SOURCE_FPS = 1 + }; + + enum MaskLoopMode { + MASK_LOOP_PLAY_ONCE = 0, + MASK_LOOP_REPEAT = 1, + MASK_LOOP_PING_PONG = 2 + }; + + int mask_time_mode = MASK_TIME_SOURCE_FPS; ///< How effect frames map to mask source frames. + int mask_loop_mode = MASK_LOOP_PLAY_ONCE; ///< Behavior when mask range reaches the end. /// Display effect information in the standard output stream (stdout) void DisplayInfo(std::ostream* out=&std::cout); @@ -112,13 +168,23 @@ namespace openshot /// Generate JSON object of base properties (recommended to be used by all effects) Json::Value BasePropertiesJSON(int64_t requested_frame) const; + /// Apply effect processing with common mask support (if enabled). + std::shared_ptr ProcessFrame(std::shared_ptr frame, int64_t frame_number); + + /// Get the common mask reader. + ReaderBase* MaskReader() { return mask_reader; } + const ReaderBase* MaskReader() const { return mask_reader; } + + /// Set or replace the common mask reader. + void MaskReader(ReaderBase* new_reader); + /// Get the order that this effect should be executed. int Order() const { return order; } /// Set the order that this effect should be executed. void Order(int new_order) { order = new_order; } - virtual ~EffectBase() = default; + virtual ~EffectBase(); }; } diff --git a/src/Enums.h b/src/Enums.h index cea6a1a2..8f3cb324 100644 --- a/src/Enums.h +++ b/src/Enums.h @@ -168,7 +168,8 @@ enum ChromaKeyMethod CHROMAKEY_CIE_LCH_H, ///< Difference between CIE LCH(ab) hues CHROMAKEY_CIE_DISTANCE, ///< CIEDE2000 perceptual difference CHROMAKEY_YCBCR, ///< YCbCr vector difference of CbCr - CHROMAKEY_LAST_METHOD = CHROMAKEY_YCBCR + CHROMAKEY_BASIC_SOFT, ///< BASIC metric + optional halo feathering + CHROMAKEY_LAST_METHOD = CHROMAKEY_BASIC_SOFT }; } // namespace openshot diff --git a/src/Exceptions.h b/src/Exceptions.h index 9e2f2b83..60a55c14 100644 --- a/src/Exceptions.h +++ b/src/Exceptions.h @@ -62,15 +62,21 @@ namespace openshot { { public: std::string file_path; + std::string full_message; FileExceptionBase(std::string message, std::string file_path="") - : ExceptionBase(message), file_path(file_path) { } - virtual std::string py_message() const override { - // return complete message for Python exception handling - std::string out_msg(m_message + + : ExceptionBase(message), + file_path(file_path), + full_message(message + (file_path != "" ? " for file " + file_path - : "")); - return out_msg; + : "")) { } + virtual const char* what() const noexcept override { + // Include file path in native C++ exception output (stderr / terminate). + return full_message.c_str(); + } + virtual std::string py_message() const override { + // Keep Python exception output consistent with what(). + return full_message; } }; diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 18a9084c..5d9e4a9f 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -73,6 +73,32 @@ int hw_de_on = 0; AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE; #endif +// Normalize deprecated JPEG-range YUVJ formats before creating swscale contexts. +// swscale expects non-YUVJ formats plus explicit color-range metadata. +static AVPixelFormat NormalizeDeprecatedPixFmt(AVPixelFormat pix_fmt, bool& is_full_range) { + switch (pix_fmt) { + case AV_PIX_FMT_YUVJ420P: + is_full_range = true; + return AV_PIX_FMT_YUV420P; + case AV_PIX_FMT_YUVJ422P: + is_full_range = true; + return AV_PIX_FMT_YUV422P; + case AV_PIX_FMT_YUVJ444P: + is_full_range = true; + return AV_PIX_FMT_YUV444P; + case AV_PIX_FMT_YUVJ440P: + is_full_range = true; + return AV_PIX_FMT_YUV440P; +#ifdef AV_PIX_FMT_YUVJ411P + case AV_PIX_FMT_YUVJ411P: + is_full_range = true; + return AV_PIX_FMT_YUV411P; +#endif + default: + return pix_fmt; + } +} + FFmpegReader::FFmpegReader(const std::string &path, bool inspect_reader) : FFmpegReader(path, DurationStrategy::VideoPreferred, inspect_reader) {} @@ -242,13 +268,16 @@ void FFmpegReader::Open() { // Initialize format context pFormatCtx = NULL; { - hw_de_on = (openshot::Settings::Instance()->HARDWARE_DECODER == 0 ? 0 : 1); + hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0); + hw_decode_failed = false; + hw_decode_error_count = 0; + hw_decode_succeeded = false; ZmqLogger::Instance()->AppendDebugMethod("Decode hardware acceleration settings", "hw_de_on", hw_de_on, "HARDWARE_DECODER", openshot::Settings::Instance()->HARDWARE_DECODER); } // Open video file if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, NULL) != 0) - throw InvalidFile("File could not be opened.", path); + throw InvalidFile("FFmpegReader could not open media file.", path); // Retrieve stream information if (avformat_find_stream_info(pFormatCtx, NULL) < 0) @@ -1117,6 +1146,43 @@ void FFmpegReader::UpdateVideoInfo() { ApplyDurationStrategy(); + // Normalize FFmpeg-decoded still images (e.g. JPG/JPEG) to match image-reader behavior. + // This keeps timing/flags consistent regardless of which reader path was used. + if (!info.has_single_image) { + const AVCodecID codec_id = AV_FIND_DECODER_CODEC_ID(pStream); + const bool likely_still_codec = + codec_id == AV_CODEC_ID_MJPEG || + codec_id == AV_CODEC_ID_PNG || + codec_id == AV_CODEC_ID_BMP || + codec_id == AV_CODEC_ID_TIFF || + codec_id == AV_CODEC_ID_WEBP || + codec_id == AV_CODEC_ID_JPEG2000; + const bool likely_image_demuxer = + pFormatCtx && pFormatCtx->iformat && pFormatCtx->iformat->name && + strstr(pFormatCtx->iformat->name, "image2"); + const bool has_attached_pic = HasAlbumArt(); + const bool single_frame_stream = + (pStream && pStream->nb_frames > 0 && pStream->nb_frames <= 1); + const bool single_frame_clip = info.video_length <= 1; + + const bool is_still_image_video = + has_attached_pic || + ((single_frame_stream || single_frame_clip) && + (likely_still_codec || likely_image_demuxer)); + + if (is_still_image_video) { + info.has_single_image = true; + + // Only force long duration for standalone images. For audio + attached-art + // files, keep stream-derived duration so the cover image spans the audio. + if (audioStream < 0) { + record_duration(video_stream_duration_seconds, 60 * 60 * 1); // 1 hour duration + } + + ApplyDurationStrategy(); + } + } + // Add video metadata (if any) AVDictionaryEntry *tag = NULL; while ((tag = av_dict_get(pStream->metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { @@ -1154,7 +1220,6 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame", requested_frame); - // Return the cached frame return frame; } else { @@ -1167,7 +1232,6 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetFrame", "returned cached frame on 2nd look", requested_frame); - } else { // Frame is not in cache // Reset seek count @@ -1257,6 +1321,9 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { (info.has_video && !packet && !packet_status.video_eof)) { // Process Video Packet ProcessVideoPacket(requested_frame); + if (ReopenWithoutHardwareDecode(requested_frame)) { + continue; + } } // Audio packet if ((info.has_audio && packet && packet->stream_index == audioStream) || @@ -1431,6 +1498,26 @@ int FFmpegReader::GetNextPacket() { // Get an AVFrame (if any) bool FFmpegReader::GetAVFrame() { int frameFinished = 0; + auto note_hw_decode_failure = [&](int err, const char* stage) { +#if USE_HW_ACCEL + if (!hw_de_on || !hw_de_supported || force_sw_decode) { + return; + } + if (err == AVERROR_INVALIDDATA && packet_status.video_decoded == 0) { + hw_decode_error_count++; + ZmqLogger::Instance()->AppendDebugMethod( + std::string("FFmpegReader::GetAVFrame (hardware decode failure candidate during ") + stage + ")", + "error_count", hw_decode_error_count, + "error", err); + if (hw_decode_error_count >= 3) { + hw_decode_failed = true; + } + } +#else + (void) err; + (void) stage; +#endif + }; // Decode video frame AVFrame *next_frame = AV_ALLOCATE_FRAME(); @@ -1455,6 +1542,7 @@ bool FFmpegReader::GetAVFrame() { #endif // USE_HW_ACCEL if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts); + note_hw_decode_failure(send_packet_err, "send_packet"); if (send_packet_err == AVERROR(EAGAIN)) { hold_packet = true; ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: AVERROR(EAGAIN): user must read output with avcodec_receive_frame()", "send_packet_pts", send_packet_pts); @@ -1471,6 +1559,7 @@ bool FFmpegReader::GetAVFrame() { // Even if the above avcodec_send_packet failed to send, // we might still need to receive a packet. int receive_frame_err = 0; + AVFrame *decoded_frame = next_frame; AVFrame *next_frame2; #if USE_HW_ACCEL if (hw_de_on && hw_de_supported) { @@ -1487,6 +1576,7 @@ bool FFmpegReader::GetAVFrame() { if (receive_frame_err != 0) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (receive frame: frame not ready yet from decoder [\" + av_err2string(receive_frame_err) + \"])", "receive_frame_err", receive_frame_err, "send_packet_pts", send_packet_pts); + note_hw_decode_failure(receive_frame_err, "receive_frame"); if (receive_frame_err == AVERROR_EOF) { ZmqLogger::Instance()->AppendDebugMethod( @@ -1517,44 +1607,93 @@ bool FFmpegReader::GetAVFrame() { if (hw_de_on && hw_de_supported) { int err; if (next_frame2->format == hw_de_av_pix_fmt) { - next_frame->format = AV_PIX_FMT_YUV420P; - if ((err = av_hwframe_transfer_data(next_frame,next_frame2,0)) < 0) { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to transfer data to output frame)", "hw_de_on", hw_de_on); + if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::GetAVFrame (Failed to transfer data to output frame)", + "hw_de_on", hw_de_on, + "error", err); + note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_transfer"); + break; } - if ((err = av_frame_copy_props(next_frame,next_frame2)) < 0) { - ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Failed to copy props to output frame)", "hw_de_on", hw_de_on); + if ((err = av_frame_copy_props(next_frame, next_frame2)) < 0) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::GetAVFrame (Failed to copy props to output frame)", + "hw_de_on", hw_de_on, + "error", err); + note_hw_decode_failure(AVERROR_INVALIDDATA, "hwframe_copy_props"); + break; } + if (next_frame->format == AV_PIX_FMT_NONE) { + next_frame->format = pCodecCtx->sw_pix_fmt; + } + if (next_frame->width <= 0) { + next_frame->width = next_frame2->width; + } + if (next_frame->height <= 0) { + next_frame->height = next_frame2->height; + } + decoded_frame = next_frame; + } else { + // Some hardware decoders can still return software-readable frames. + decoded_frame = next_frame2; } } else #endif // USE_HW_ACCEL { // No hardware acceleration used -> no copy from GPU memory needed - next_frame = next_frame2; + decoded_frame = next_frame2; + } + + if (!decoded_frame->data[0]) { + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::GetAVFrame (Decoded frame missing image data)", + "format", decoded_frame->format, + "width", decoded_frame->width, + "height", decoded_frame->height); + note_hw_decode_failure(AVERROR_INVALIDDATA, "decoded_frame_empty"); + break; } // TODO also handle possible further frames // Use only the first frame like avcodec_decode_video2 frameFinished = 1; + hw_decode_error_count = 0; +#if USE_HW_ACCEL + if (hw_de_on && hw_de_supported && !force_sw_decode) { + hw_decode_succeeded = true; + } +#endif packet_status.video_decoded++; // Allocate image (align 32 for simd) - if (AV_ALLOCATE_IMAGE(pFrame, (AVPixelFormat)(pStream->codecpar->format), info.width, info.height) <= 0) { + AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format); + if (decoded_pix_fmt == AV_PIX_FMT_NONE) + decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format); + if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, info.width, info.height) <= 0) { throw OutOfMemory("Failed to allocate image buffer", path); } - av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)next_frame->data, next_frame->linesize, - (AVPixelFormat)(pStream->codecpar->format), info.width, info.height); + av_image_copy(pFrame->data, pFrame->linesize, (const uint8_t**)decoded_frame->data, decoded_frame->linesize, + decoded_pix_fmt, info.width, info.height); + pFrame->format = decoded_pix_fmt; + pFrame->width = info.width; + pFrame->height = info.height; + pFrame->color_range = decoded_frame->color_range; + pFrame->colorspace = decoded_frame->colorspace; + pFrame->color_primaries = decoded_frame->color_primaries; + pFrame->color_trc = decoded_frame->color_trc; + pFrame->chroma_location = decoded_frame->chroma_location; // Get display PTS from video frame, often different than packet->pts. // Sending packets to the decoder (i.e. packet->pts) is async, // and retrieving packets from the decoder (frame->pts) is async. In most decoders // sending and retrieving are separated by multiple calls to this method. - if (next_frame->pts != AV_NOPTS_VALUE) { + if (decoded_frame->pts != AV_NOPTS_VALUE) { // This is the current decoded frame (and should be the pts used) for // processing this data - video_pts = next_frame->pts; - } else if (next_frame->pkt_dts != AV_NOPTS_VALUE) { + video_pts = decoded_frame->pts; + } else if (decoded_frame->pkt_dts != AV_NOPTS_VALUE) { // Some videos only set this timestamp (fallback) - video_pts = next_frame->pkt_dts; + video_pts = decoded_frame->pkt_dts; } ZmqLogger::Instance()->AppendDebugMethod( @@ -1564,7 +1703,7 @@ bool FFmpegReader::GetAVFrame() { break; } #if USE_HW_ACCEL - if (hw_de_on && hw_de_supported) { + if (hw_de_on && hw_de_supported && next_frame2 != next_frame) { AV_FREE_FRAME(&next_frame2); } #endif // USE_HW_ACCEL @@ -1591,6 +1730,41 @@ bool FFmpegReader::GetAVFrame() { return frameFinished; } +bool FFmpegReader::ReopenWithoutHardwareDecode(int64_t requested_frame) { +#if USE_HW_ACCEL + if (!hw_decode_failed || force_sw_decode) { + return false; + } + + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::ReopenWithoutHardwareDecode (falling back to software decode)", + "requested_frame", requested_frame, + "video_packets_read", packet_status.video_read, + "video_packets_decoded", packet_status.video_decoded, + "hw_decode_error_count", hw_decode_error_count); + + force_sw_decode = true; + hw_decode_failed = false; + hw_decode_error_count = 0; + + Close(); + Open(); + Seek(requested_frame); + return true; +#else + (void) requested_frame; + return false; +#endif +} + +bool FFmpegReader::HardwareDecodeSuccessful() const { +#if USE_HW_ACCEL + return hw_decode_succeeded; +#else + return false; +#endif +} + // Check the current seek position and determine if we need to seek again bool FFmpegReader::CheckSeek() { // Are we seeking for a specific frame? @@ -1691,9 +1865,15 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::ProcessVideoPacket (Before)", "requested_frame", requested_frame, "current_frame", current_frame); // Init some things local (for OpenMP) - PixelFormat pix_fmt = AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx); - int height = info.height; - int width = info.width; + AVPixelFormat decoded_pix_fmt = (pFrame && pFrame->format != AV_PIX_FMT_NONE) + ? static_cast(pFrame->format) + : AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx); + bool src_full_range = (pFrame && pFrame->color_range == AVCOL_RANGE_JPEG); + AVPixelFormat src_pix_fmt = NormalizeDeprecatedPixFmt(decoded_pix_fmt, src_full_range); + int src_width = (pFrame && pFrame->width > 0) ? pFrame->width : info.width; + int src_height = (pFrame && pFrame->height > 0) ? pFrame->height : info.height; + int height = src_height; + int width = src_width; int64_t video_length = info.video_length; // Create or reuse a RGB Frame (since most videos are not in RGB, we must convert it) @@ -1766,7 +1946,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } // Determine if image needs to be scaled (for performance reasons) - int original_height = height; + int original_height = src_height; if (max_width != 0 && max_height != 0 && max_width < width && max_height < height) { // Override width and height (but maintain aspect ratio) float ratio = float(width) / float(height); @@ -1800,19 +1980,64 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) { scale_mode = SWS_BICUBIC; } - img_convert_ctx = sws_getCachedContext(img_convert_ctx, info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx), width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL); + img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL); if (!img_convert_ctx) throw OutOfMemory("Failed to initialize sws context", path); + const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT); + const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT); + const int dst_full_range = 1; // RGB outputs are full-range + sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0, + dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16); + + if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) { +#if USE_HW_ACCEL + if (hw_de_on && hw_de_supported && !force_sw_decode) { + hw_decode_failed = true; + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::ProcessVideoPacket (Invalid source frame; forcing software fallback)", + "requested_frame", requested_frame, + "current_frame", current_frame, + "src_pix_fmt", src_pix_fmt, + "src_width", src_width, + "src_height", src_height); + } +#endif + if (pFrame) { + RemoveAVFrame(pFrame); + pFrame = NULL; + } + return; + } // Resize / Convert to RGB - sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, + const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, original_height, pFrameRGB->data, pFrameRGB->linesize); + if (scaled_lines <= 0) { +#if USE_HW_ACCEL + if (hw_de_on && hw_de_supported && !force_sw_decode) { + hw_decode_failed = true; + ZmqLogger::Instance()->AppendDebugMethod( + "FFmpegReader::ProcessVideoPacket (sws_scale failed; forcing software fallback)", + "requested_frame", requested_frame, + "current_frame", current_frame, + "scaled_lines", scaled_lines, + "src_pix_fmt", src_pix_fmt, + "src_width", src_width, + "src_height", src_height); + } +#endif + free(buffer); + AV_RESET_FRAME(pFrameRGB); + RemoveAVFrame(pFrame); + pFrame = NULL; + return; + } // Create or get the existing frame object std::shared_ptr f = CreateFrame(current_frame); // Add Image data to frame - if (!ffmpeg_has_alpha(AV_GET_CODEC_PIXEL_FORMAT(pStream, pCodecCtx))) { + if (!ffmpeg_has_alpha(src_pix_fmt)) { // Add image with no alpha channel, Speed optimization f->AddImage(width, height, bytes_per_pixel, QImage::Format_RGBA8888_Premultiplied, buffer); } else { @@ -2263,7 +2488,7 @@ void FFmpegReader::UpdatePTSOffset() { // Video packet if (!has_video_pts && packet->stream_index == videoStream) { // Get the video packet start time (in seconds) - video_pts_offset_seconds = 0.0 - (video_pts * info.video_timebase.ToDouble()); + video_pts_offset_seconds = 0.0 - (pts * info.video_timebase.ToDouble()); // Is timestamp close to zero (within X seconds) // Ignore wildly invalid timestamps (i.e. -234923423423) @@ -2283,20 +2508,17 @@ void FFmpegReader::UpdatePTSOffset() { } } - // Do we have all valid timestamps to determine PTS offset? - if (has_video_pts && has_audio_pts) { - // Set PTS Offset to the smallest offset - // [ video timestamp ] - // [ audio timestamp ] - // - // ** SHIFT TIMESTAMPS TO ZERO ** - // - //[ video timestamp ] - // [ audio timestamp ] - // - // Since all offsets are negative at this point, we want the max value, which - // represents the closest to zero - pts_offset_seconds = std::max(video_pts_offset_seconds, audio_pts_offset_seconds); + // Choose timestamp origin: + // - If video exists, anchor timeline frame mapping to video start. + // This avoids AAC priming / audio preroll shifting video frame 1 to frame 2. + // - If no video exists (audio-only readers), use audio start. + if (info.has_video && has_video_pts) { + pts_offset_seconds = video_pts_offset_seconds; + } else if (!info.has_video && has_audio_pts) { + pts_offset_seconds = audio_pts_offset_seconds; + } else if (has_video_pts && has_audio_pts) { + // Fallback when stream flags are unusual but both timestamps exist. + pts_offset_seconds = video_pts_offset_seconds; } } @@ -2516,7 +2738,8 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { "frame_pts_seconds", frame_pts_seconds, "video_pts_seconds", video_pts_seconds, "recent_pts_diff", recent_pts_diff); - if (info.has_video && !f->has_image_data) { + if (info.has_video && !f->has_image_data && + (packet_status.video_eof || packet_status.end_of_file)) { // Frame has no image data. Prefer timeline-previous frames to preserve // visual order, especially when decode/prefetch is out-of-order. std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); @@ -2581,6 +2804,35 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { "end_of_file", packet_status.end_of_file); // Check if working frame is final + if (info.has_video && !f->has_image_data + && !packet_status.end_of_file && !is_seek_trash) { + if (info.has_single_image) { + // For still-image video (including attached cover art), reuse the most + // recent image so playback does not stall waiting for video EOF. + std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); + if (previous_frame_instance && previous_frame_instance->has_image_data) { + f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); + } + if (!f->has_image_data + && last_final_video_frame + && last_final_video_frame->has_image_data + && last_final_video_frame->number <= f->number) { + f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); + } + if (!f->has_image_data + && last_video_frame + && last_video_frame->has_image_data + && last_video_frame->number <= f->number) { + f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); + } + } + + // Do not finalize non-EOF video frames without decoded image data. + // This prevents repeated previous-frame fallbacks being cached as real frames. + if (!f->has_image_data) { + continue; + } + } if ((!packet_status.end_of_file && is_video_ready && is_audio_ready) || packet_status.end_of_file || is_seek_trash) { // Debug output ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::CheckWorkingFrames (mark frame as final)", @@ -2772,10 +3024,4 @@ void FFmpegReader::SetJsonValue(const Json::Value root) { duration_strategy = DurationStrategy::LongestStream; } } - - // Re-Open path, and re-init everything (if needed) - if (is_open) { - Close(); - Open(); - } } diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index 7b42c9d4..6417bb1d 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -165,6 +165,10 @@ namespace openshot { AVFrame *pFrameRGB_cached = nullptr; ///< Temporary frame used for video conversion int hw_de_supported = 0; // Is set by FFmpegReader + bool force_sw_decode = false; + bool hw_decode_failed = false; + int hw_decode_error_count = 0; + bool hw_decode_succeeded = false; #if USE_HW_ACCEL AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE; AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE; @@ -198,6 +202,9 @@ namespace openshot { /// Get an AVFrame (if any) bool GetAVFrame(); + /// Reopen the current reader with software decode after hardware decode fails + bool ReopenWithoutHardwareDecode(int64_t requested_frame); + /// Get the next packet (if any) int GetNextPacket(); @@ -285,6 +292,9 @@ namespace openshot { /// Determine if reader is open or closed bool IsOpen() override { return is_open; }; + /// Return true if hardware decode was requested and successfully produced at least one frame + bool HardwareDecodeSuccessful() const override; + /// Return the type name of the class std::string Name() override { return "FFmpegReader"; }; diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 26379041..d093e305 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -79,7 +79,7 @@ FFmpegWriter::FFmpegWriter(const std::string& path) : initial_audio_input_frame_size(0), img_convert_ctx(NULL), video_codec_ctx(NULL), audio_codec_ctx(NULL), is_writing(false), video_timestamp(0), audio_timestamp(0), original_sample_rate(0), original_channels(0), avr(NULL), avr_planar(NULL), is_open(false), prepare_streams(false), - write_header(false), write_trailer(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) { + write_header(false), write_trailer(false), allow_b_frames(false), audio_encoder_buffer_size(0), audio_encoder_buffer(NULL) { // Disable audio & video (so they can be independently enabled) info.has_audio = false; @@ -368,7 +368,7 @@ void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string va // Was option found? if (option || (name == "g" || name == "qmin" || name == "qmax" || name == "max_b_frames" || name == "mb_decision" || name == "level" || name == "profile" || name == "slices" || name == "rc_min_rate" || name == "rc_max_rate" || - name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp")) { + name == "rc_buffer_size" || name == "crf" || name == "cqp" || name == "qp" || name == "allow_b_frames")) { // Check for specific named options if (name == "g") // Set gop_size @@ -386,6 +386,11 @@ void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string va // Maximum number of B-frames between non-B-frames convert >> c->max_b_frames; + else if (name == "allow_b_frames") + // Preserve configured B-frames for codecs that support them. + // Values: 1/true/yes/on to enable, everything else disables. + allow_b_frames = (value == "1" || value == "true" || value == "yes" || value == "on"); + else if (name == "mb_decision") // Macroblock decision mode convert >> c->mb_decision; @@ -1468,8 +1473,12 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { if (!codec) throw InvalidCodec("Could not find codec", path); - /* Force max_b_frames to 0 in some cases (i.e. for mjpeg image sequences */ - if (video_codec_ctx->max_b_frames && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO && video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO) + /* Legacy behavior: force max_b_frames to 0 for many codecs. + * This can be disabled via SetOption(VIDEO_STREAM, "allow_b_frames", "1"). */ + if (!allow_b_frames && video_codec_ctx->max_b_frames && + video_codec_ctx->codec_id != AV_CODEC_ID_MPEG4 && + video_codec_ctx->codec_id != AV_CODEC_ID_MPEG1VIDEO && + video_codec_ctx->codec_id != AV_CODEC_ID_MPEG2VIDEO) video_codec_ctx->max_b_frames = 0; // Init options diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 4fab6a77..fdfd874d 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -124,6 +124,7 @@ namespace openshot { bool prepare_streams; bool write_header; bool write_trailer; + bool allow_b_frames; AVFormatContext* oc; AVStream *audio_st, *video_st; diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index 5e678b5b..937f828b 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -800,13 +800,6 @@ void FrameMapper::SetJsonValue(const Json::Value root) { // Set parent data ReaderBase::SetJsonValue(root); - - // Re-Open path, and re-init everything (if needed) - if (reader) { - - Close(); - Open(); - } } // Change frame rate or audio mapping details diff --git a/src/ImageReader.cpp b/src/ImageReader.cpp index e3342a41..0b9c4abd 100644 --- a/src/ImageReader.cpp +++ b/src/ImageReader.cpp @@ -49,7 +49,7 @@ void ImageReader::Open() } catch (const Magick::Exception& e) { // raise exception - throw InvalidFile("File could not be opened.", path); + throw InvalidFile("ImageReader could not open image file.", path); } // Update image properties diff --git a/src/Profiles.cpp b/src/Profiles.cpp index d76557d4..088a6b1f 100644 --- a/src/Profiles.cpp +++ b/src/Profiles.cpp @@ -129,13 +129,13 @@ Profile::Profile(std::string path) { catch (const std::exception& e) { // Error parsing profile file - throw InvalidFile("Profile could not be found or loaded (or is invalid).", path); + throw InvalidFile("Profile file could not be parsed (invalid format or values).", path); } // Throw error if file was not read if (!read_file) // Error parsing profile file - throw InvalidFile("Profile could not be found or loaded (or is invalid).", path); + throw InvalidFile("Profile file could not be found or opened.", path); } // Return a formatted FPS diff --git a/src/Qt/PlayerPrivate.cpp b/src/Qt/PlayerPrivate.cpp index 222bfd5e..bacaef73 100644 --- a/src/Qt/PlayerPrivate.cpp +++ b/src/Qt/PlayerPrivate.cpp @@ -74,9 +74,13 @@ namespace openshot // Pausing Code (which re-syncs audio/video times) // - If speed is zero or speed changes // - If pre-roll is not ready (This should allow scrubbing of the timeline without waiting on pre-roll) - if ((speed == 0 && video_position == last_video_position) || - (speed != 0 && last_speed != speed) || - (speed != 0 && !is_dirty && !videoCache->isReady())) + bool wait_paused_hold = (speed == 0 && video_position == last_video_position); + bool wait_speed_change = (speed != 0 && last_speed != speed); + bool cache_ready = videoCache->isReady(); + bool wait_preroll = (speed != 0 && !is_dirty && !cache_ready); + bool should_wait = (wait_paused_hold || wait_speed_change || wait_preroll); + + if (should_wait) { // Sleep for a fraction of frame duration std::this_thread::sleep_for(frame_duration / 4); @@ -97,7 +101,16 @@ namespace openshot // Set the video frame on the video thread and render frame videoPlayback->frame = frame; + videoPlayback->rendered.reset(); videoPlayback->render.signal(); + // Keep decode/position advancement aligned with actual preview updates. + // This avoids occasional "silent advance then jump" behavior when + // preroll transitions and rendering are briefly out-of-sync. + const int render_wait_ms = std::max( + 1, + static_cast(frame_duration.count() / 1000.0 * 2.0) + ); + videoPlayback->rendered.wait(render_wait_ms); // Keep track of the last displayed frame last_video_position = video_position; @@ -116,6 +129,12 @@ namespace openshot // Protect against invalid or too-long sleep times std::this_thread::sleep_for(max_sleep); } + } else { + // If we're behind schedule (e.g. preroll/render stall), do not + // burst through delayed frames. Resync timing baseline so + // playback continues smoothly at normal cadence. + start_time = std::chrono::time_point_cast(current_time); + playback_frames = 0; } } } @@ -150,8 +169,8 @@ namespace openshot // Increment playback frames (always in the positive direction) playback_frames += std::abs(speed); - // Update cache on which frame was retrieved - videoCache->Seek(video_position); + // Update playhead hint for cache window tracking without triggering seek behavior. + videoCache->NotifyPlaybackPosition(video_position); // return frame from reader return reader->GetFrame(video_position); @@ -170,6 +189,10 @@ namespace openshot { video_position = new_position; last_video_position = 0; + // Drop local frame reference so same-frame refreshes cannot reuse stale + // content after timeline/clip property updates. + frame.reset(); + // Always force immediate refresh after seek/update, even while playing. is_dirty = true; } diff --git a/src/Qt/VideoCacheThread.cpp b/src/Qt/VideoCacheThread.cpp index 6513e692..ad80e33a 100644 --- a/src/Qt/VideoCacheThread.cpp +++ b/src/Qt/VideoCacheThread.cpp @@ -30,6 +30,8 @@ namespace openshot , last_dir(1) // assume forward (+1) on first launch , userSeeked(false) , preroll_on_next_fill(false) + , clear_cache_on_next_fill(false) + , scrub_active(false) , requested_display_frame(1) , current_display_frame(1) , cached_frame_count(0) @@ -38,6 +40,8 @@ namespace openshot , reader(nullptr) , force_directional_cache(false) , last_cached_index(0) + , seen_timeline_cache_epoch(0) + , timeline_cache_epoch_initialized(false) { } @@ -59,7 +63,7 @@ namespace openshot } const int64_t cached_index = last_cached_index.load(); - const int64_t playhead = requested_display_frame.load(); + int64_t playhead = requested_display_frame.load(); int dir = computeDirection(); // Near timeline boundaries, don't require more pre-roll than can exist. @@ -73,6 +77,7 @@ namespace openshot if (max_frame < 1) { return false; } + playhead = clampToTimelineRange(playhead, max_frame); int64_t required_ahead = ready_min; int64_t available_ahead = (dir > 0) @@ -92,6 +97,8 @@ namespace openshot if (new_speed != 0) { last_speed.store(new_speed); last_dir.store(new_speed > 0 ? 1 : -1); + // Leaving paused/scrub context: resume normal cache behavior. + scrub_active.store(false); } speed.store(new_speed); } @@ -126,45 +133,176 @@ namespace openshot return !isThreadRunning(); } + void VideoCacheThread::Reader(ReaderBase* new_reader) + { + std::lock_guard guard(seek_state_mutex); + reader = new_reader; + seen_timeline_cache_epoch = 0; + timeline_cache_epoch_initialized = false; + Play(); + } + void VideoCacheThread::Seek(int64_t new_position, bool start_preroll) { + const int64_t timeline_end = resolveTimelineEnd(); + const int64_t clamped_new_position = clampToTimelineRange(new_position, timeline_end); + const int64_t current_requested = requested_display_frame.load(); + bool should_mark_seek = false; bool should_preroll = false; int64_t new_cached_count = cached_frame_count.load(); + bool entering_scrub = false; + bool leaving_scrub = false; + bool cache_contains = false; + bool should_clear_cache = false; + CacheBase* cache = reader ? reader->GetCache() : nullptr; + const bool same_frame_refresh = (new_position == current_requested); + if (cache) { + cache_contains = cache->Contains(clamped_new_position); + } if (start_preroll) { - should_mark_seek = true; - CacheBase* cache = reader ? reader->GetCache() : nullptr; - - if (cache && !cache->Contains(new_position)) - { - // If user initiated seek, and current frame not found ( - if (Timeline* timeline = dynamic_cast(reader)) { - timeline->ClearAllCache(); + if (same_frame_refresh) { + const bool is_paused = (speed.load() == 0); + if (is_paused) { + const bool was_scrubbing = scrub_active.load(); + if (was_scrubbing && cache && cache_contains) { + // Preserve in-range cache for paused scrub preview -> same-frame commit. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + new_cached_count = cache->Count(); + } else { + // Paused same-frame edit refresh: force full cache refresh. + if (Timeline* timeline = dynamic_cast(reader)) { + timeline->ClearAllCache(); + } + new_cached_count = 0; + should_mark_seek = true; + should_preroll = true; + should_clear_cache = false; + } + } else { + // Same-frame refresh during playback should stay lightweight. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + if (cache && cache_contains) { + cache->Remove(clamped_new_position); + } + if (cache) { + new_cached_count = cache->Count(); + } + } + } else { + if (cache && !cache_contains) { + should_mark_seek = true; + // Uncached commit seek: defer cache clear to cache thread loop. + new_cached_count = 0; + should_preroll = true; + should_clear_cache = true; + } + else if (cache) + { + // In-range commit seek preserves cache window/baseline. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + new_cached_count = cache->Count(); + } else { + // No cache object to query: use normal seek behavior. + should_mark_seek = true; } - new_cached_count = 0; - should_preroll = true; } - else if (cache) - { - new_cached_count = cache->Count(); + leaving_scrub = true; + } + else { + // Non-preroll seeks cover paused scrubbing and live playback refresh. + const bool is_paused = (speed.load() == 0); + if (is_paused && same_frame_refresh) { + // Same-frame paused refresh updates only that frame. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + if (cache && cache_contains) { + cache->Remove(clamped_new_position); + } + if (cache) { + new_cached_count = cache->Count(); + } + leaving_scrub = true; + } + else if (is_paused) { + if (cache && !cache_contains) { + should_mark_seek = true; + new_cached_count = 0; + should_clear_cache = true; + } + else if (cache) { + // In-range paused seek preserves cache continuity. + should_mark_seek = false; + new_cached_count = cache->Count(); + } else { + should_mark_seek = true; + } + entering_scrub = true; + } else { + // During playback, keep seek/scrub side effects minimal. + should_mark_seek = false; + should_preroll = false; + should_clear_cache = false; + if (cache) { + new_cached_count = cache->Count(); + } + leaving_scrub = true; } } { std::lock_guard guard(seek_state_mutex); + // Reset readiness baseline only when rebuilding cache. + const int dir = computeDirection(); + if (should_mark_seek || should_preroll || should_clear_cache) { + last_cached_index.store(clamped_new_position - dir); + } requested_display_frame.store(new_position); cached_frame_count.store(new_cached_count); - if (start_preroll) { - preroll_on_next_fill.store(should_preroll); - userSeeked.store(should_mark_seek); + preroll_on_next_fill.store(should_preroll); + // Clear behavior follows the latest seek intent. + clear_cache_on_next_fill.store(should_clear_cache); + userSeeked.store(should_mark_seek); + if (entering_scrub) { + scrub_active.store(true); + } + if (leaving_scrub) { + scrub_active.store(false); } } } void VideoCacheThread::Seek(int64_t new_position) { - Seek(new_position, false); + NotifyPlaybackPosition(new_position); + } + + void VideoCacheThread::NotifyPlaybackPosition(int64_t new_position) + { + if (new_position <= 0) { + return; + } + if (scrub_active.load()) { + return; + } + + int64_t new_cached_count = cached_frame_count.load(); + if (CacheBase* cache = reader ? reader->GetCache() : nullptr) { + new_cached_count = cache->Count(); + } + { + std::lock_guard guard(seek_state_mutex); + requested_display_frame.store(new_position); + cached_frame_count.store(new_cached_count); + } } int VideoCacheThread::computeDirection() const @@ -216,11 +354,39 @@ namespace openshot return min_frames; } + int64_t VideoCacheThread::resolveTimelineEnd() const + { + if (!reader) { + return 0; + } + int64_t timeline_end = reader->info.video_length; + if (auto* timeline = dynamic_cast(reader)) { + const int64_t timeline_max = timeline->GetMaxFrame(); + if (timeline_max > 0) { + timeline_end = timeline_max; + } + } + return timeline_end; + } + + int64_t VideoCacheThread::clampToTimelineRange(int64_t frame, int64_t timeline_end) const + { + if (timeline_end < 1) { + return frame; + } + return std::clamp(frame, 1, timeline_end); + } + bool VideoCacheThread::clearCacheIfPaused(int64_t playhead, bool paused, CacheBase* cache) { - if (paused && !cache->Contains(playhead)) { + const int64_t timeline_end = resolveTimelineEnd(); + int64_t cache_playhead = playhead; + if (reader) { + cache_playhead = clampToTimelineRange(playhead, timeline_end); + } + if (paused && !cache->Contains(cache_playhead)) { // If paused and playhead not in cache, clear everything if (Timeline* timeline = dynamic_cast(reader)) { timeline->ClearAllCache(); @@ -257,10 +423,12 @@ namespace openshot int64_t window_begin, int64_t window_end, int dir, - ReaderBase* reader) + ReaderBase* reader, + int64_t max_frames_to_fetch) { bool window_full = true; int64_t next_frame = last_cached_index.load() + dir; + int64_t fetched_this_pass = 0; // Advance from last_cached_index toward window boundary while ((dir > 0 && next_frame <= window_end) || @@ -280,6 +448,7 @@ namespace openshot auto framePtr = reader->GetFrame(next_frame); cache->Add(framePtr); cached_frame_count.store(cache->Count()); + ++fetched_this_pass; } catch (const OutOfBoundsFrame&) { break; @@ -292,6 +461,12 @@ namespace openshot last_cached_index.store(next_frame); next_frame += dir; + + // In active playback, avoid long uninterrupted prefetch bursts + // that can delay player thread frame retrieval. + if (max_frames_to_fetch > 0 && fetched_this_pass >= max_frames_to_fetch) { + break; + } } return window_full; @@ -305,6 +480,22 @@ namespace openshot while (!threadShouldExit()) { Settings* settings = Settings::Instance(); CacheBase* cache = reader ? reader->GetCache() : nullptr; + Timeline* timeline = dynamic_cast(reader); + + // Process deferred clears even when caching is currently disabled + // (e.g. active scrub mode), so stale ranges are removed promptly. + bool should_clear_cache = clear_cache_on_next_fill.exchange(false); + if (should_clear_cache && timeline) { + const int dir_on_clear = computeDirection(); + const int64_t clear_playhead = clampToTimelineRange( + requested_display_frame.load(), resolveTimelineEnd()); + timeline->ClearAllCache(); + cached_frame_count.store(0); + // Reset ready baseline immediately after clear. Otherwise a + // stale last_cached_index from the old cache window can make + // isReady() report true before new preroll is actually filled. + last_cached_index.store(clear_playhead - dir_on_clear); + } // If caching disabled or no reader, mark cache as ready and sleep briefly if (!settings->ENABLE_PLAYBACK_CACHING || !cache) { @@ -317,13 +508,13 @@ namespace openshot // init local vars min_frames_ahead.store(settings->VIDEO_CACHE_MIN_PREROLL_FRAMES); - Timeline* timeline = dynamic_cast(reader); if (!timeline) { std::this_thread::sleep_for(double_micro_sec(50000)); continue; } - int64_t timeline_end = timeline->GetMaxFrame(); - int64_t playhead = requested_display_frame.load(); + int64_t timeline_end = resolveTimelineEnd(); + int64_t raw_playhead = requested_display_frame.load(); + int64_t playhead = clampToTimelineRange(raw_playhead, timeline_end); bool paused = (speed.load() == 0); int64_t preroll_frames = computePrerollFrames(settings); @@ -335,6 +526,27 @@ namespace openshot last_dir.store(dir); } + // If timeline-side cache invalidation occurred (e.g. ApplyJsonDiff / SetJson), + // restart fill from the active playhead window so invalidated gaps self-heal. + if (timeline) { + bool epoch_changed = false; + { + std::lock_guard guard(seek_state_mutex); + const uint64_t timeline_epoch = timeline->CacheEpoch(); + if (!timeline_cache_epoch_initialized) { + seen_timeline_cache_epoch = timeline_epoch; + timeline_cache_epoch_initialized = true; + } + else if (timeline_epoch != seen_timeline_cache_epoch) { + seen_timeline_cache_epoch = timeline_epoch; + epoch_changed = true; + } + } + if (epoch_changed) { + handleUserSeek(playhead, dir); + } + } + // Compute bytes_per_frame, max_bytes, and capacity once int64_t bytes_per_frame = getBytes( (timeline->preview_width ? timeline->preview_width : reader->info.width), @@ -357,7 +569,8 @@ namespace openshot bool use_preroll = false; { std::lock_guard guard(seek_state_mutex); - playhead = requested_display_frame.load(); + raw_playhead = requested_display_frame.load(); + playhead = clampToTimelineRange(raw_playhead, timeline_end); did_user_seek = userSeeked.load(); use_preroll = preroll_on_next_fill.load(); if (did_user_seek) { @@ -366,7 +579,10 @@ namespace openshot } } if (did_user_seek) { - if (use_preroll) { + // During active playback, prioritize immediate forward readiness + // from the playhead. Use directional preroll offset only while + // paused/scrubbing contexts. + if (use_preroll && paused) { handleUserSeekWithPreroll(playhead, dir, timeline_end, preroll_frames); } else { @@ -395,6 +611,22 @@ namespace openshot } } + // If a clear was requested by a seek that arrived after the loop + // began, apply it now before any additional prefetch work. This + // avoids "build then suddenly clear" behavior during playback. + bool should_clear_mid_loop = clear_cache_on_next_fill.exchange(false); + if (should_clear_mid_loop && timeline) { + timeline->ClearAllCache(); + cached_frame_count.store(0); + last_cached_index.store(playhead - dir); + } + + // While user is dragging/scrubbing, skip cache prefetch work. + if (scrub_active.load()) { + std::this_thread::sleep_for(double_micro_sec(10000)); + continue; + } + // If capacity is insufficient, sleep and retry if (capacity < 1) { std::this_thread::sleep_for(double_micro_sec(50000)); @@ -411,7 +643,8 @@ namespace openshot ready_target = 0; } int64_t configured_min = settings->VIDEO_CACHE_MIN_PREROLL_FRAMES; - min_frames_ahead.store(std::min(configured_min, ready_target)); + const int64_t required_ahead = std::min(configured_min, ready_target); + min_frames_ahead.store(required_ahead); // If paused and playhead is no longer in cache, clear everything bool did_clear = clearCacheIfPaused(playhead, paused, cache); @@ -429,7 +662,21 @@ namespace openshot window_end); // Attempt to fill any missing frames in that window - bool window_full = prefetchWindow(cache, window_begin, window_end, dir, reader); + int64_t max_frames_to_fetch = -1; + if (!paused) { + // Keep cache thread responsive during playback seeks so player + // can start as soon as pre-roll is met instead of waiting for a + // full cache window pass. + max_frames_to_fetch = 8; + } + bool window_full = prefetchWindow( + cache, + window_begin, + window_end, + dir, + reader, + max_frames_to_fetch + ); // If paused and window was already full, keep playhead fresh if (paused && window_full) { diff --git a/src/Qt/VideoCacheThread.h b/src/Qt/VideoCacheThread.h index a9dc2dd6..d9ce0174 100644 --- a/src/Qt/VideoCacheThread.h +++ b/src/Qt/VideoCacheThread.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -61,7 +62,7 @@ namespace openshot /// @return The current speed (1=normal, 2=fast, –1=rewind, etc.) int getSpeed() const { return speed.load(); } - /// Seek to a specific frame (no preroll). + /// Backward-compatible alias for playback position updates (no seek side effects). void Seek(int64_t new_position); /** @@ -71,6 +72,9 @@ namespace openshot */ void Seek(int64_t new_position, bool start_preroll); + /// Update playback position without triggering seek behavior or cache invalidation. + void NotifyPlaybackPosition(int64_t new_position); + /// Start the cache thread at high priority. Returns true if it’s actually running. bool StartThread(); @@ -81,7 +85,7 @@ namespace openshot * @brief Attach a ReaderBase (e.g. Timeline, FFmpegReader) and begin caching. * @param new_reader */ - void Reader(ReaderBase* new_reader) { reader = new_reader; Play(); } + void Reader(ReaderBase* new_reader); protected: /// Thread entry point: loops until threadShouldExit() is true. @@ -125,6 +129,12 @@ namespace openshot /// @brief Compute preroll frame count from settings. int64_t computePrerollFrames(const Settings* settings) const; + /// @brief Resolve timeline end frame from reader/timeline metadata. + int64_t resolveTimelineEnd() const; + + /// @brief Clamp frame index to [1, timeline_end] when timeline_end is valid. + int64_t clampToTimelineRange(int64_t frame, int64_t timeline_end) const; + /** * @brief When paused and playhead is outside current cache, clear all frames. * @param playhead Current requested_display_frame @@ -171,7 +181,8 @@ namespace openshot int64_t window_begin, int64_t window_end, int dir, - ReaderBase* reader); + ReaderBase* reader, + int64_t max_frames_to_fetch = -1); //---------- Internal state ---------- @@ -182,6 +193,8 @@ namespace openshot std::atomic last_dir; ///< Last direction sign (+1 forward, –1 backward). std::atomic userSeeked; ///< True if Seek(..., true) was called (forces a cache reset). std::atomic preroll_on_next_fill; ///< True if next cache rebuild should include preroll offset. + std::atomic clear_cache_on_next_fill; ///< True if next cache loop should clear existing cache ranges. + std::atomic scrub_active; ///< True while user is dragging/scrubbing the playhead. std::atomic requested_display_frame; ///< Frame index the user requested. int64_t current_display_frame; ///< Currently displayed frame (unused here, reserved). @@ -192,6 +205,8 @@ namespace openshot ReaderBase* reader; ///< The source reader (e.g., Timeline, FFmpegReader). bool force_directional_cache; ///< (Reserved for future use). + uint64_t seen_timeline_cache_epoch; ///< Last observed Timeline cache invalidation epoch. + bool timeline_cache_epoch_initialized; ///< True once an initial epoch snapshot has been taken. std::atomic last_cached_index; ///< Index of the most recently cached frame. mutable std::mutex seek_state_mutex; ///< Protects coherent seek state updates/consumption. diff --git a/src/QtImageReader.cpp b/src/QtImageReader.cpp index ffd20862..5504a808 100644 --- a/src/QtImageReader.cpp +++ b/src/QtImageReader.cpp @@ -77,7 +77,7 @@ void QtImageReader::Open() if (!loaded) { // raise exception - throw InvalidFile("File could not be opened.", path.toStdString()); + throw InvalidFile("QtImageReader could not open image file.", path.toStdString()); } // Update image properties @@ -367,11 +367,4 @@ void QtImageReader::SetJsonValue(const Json::Value root) { // Set data from Json (if key is found) if (!root["path"].isNull()) path = QString::fromStdString(root["path"].asString()); - - // Re-Open path, and re-init everything (if needed) - if (is_open) - { - Close(); - Open(); - } } diff --git a/src/QtPlayer.cpp b/src/QtPlayer.cpp index 174687c9..0b5664da 100644 --- a/src/QtPlayer.cpp +++ b/src/QtPlayer.cpp @@ -164,11 +164,16 @@ namespace openshot } void QtPlayer::Seek(int64_t new_frame) + { + Seek(new_frame, true); + } + + void QtPlayer::Seek(int64_t new_frame, bool start_preroll) { // Check for seek if (reader && threads_started && new_frame > 0) { // Notify cache thread that seek has occurred - p->videoCache->Seek(new_frame, true); + p->videoCache->Seek(new_frame, start_preroll); // Notify audio thread that seek has occurred p->audioPlayback->Seek(new_frame); diff --git a/src/QtPlayer.h b/src/QtPlayer.h index 71c7df2c..60c3d556 100644 --- a/src/QtPlayer.h +++ b/src/QtPlayer.h @@ -78,6 +78,9 @@ namespace openshot /// Seek to a specific frame in the player void Seek(int64_t new_frame); + /// Seek to a specific frame, optionally triggering preroll cache rebuild. + void Seek(int64_t new_frame, bool start_preroll); + /// Set the source URL/path of this player (which will create an internal Reader) void SetSource(const std::string &source); diff --git a/src/ReaderBase.h b/src/ReaderBase.h index aca12ff2..319eda24 100644 --- a/src/ReaderBase.h +++ b/src/ReaderBase.h @@ -113,6 +113,10 @@ namespace openshot /// Determine if reader is open or closed virtual bool IsOpen() = 0; + /// Return true if hardware decode successfully produced at least one frame. + /// Readers without hardware decode support should return false. + virtual bool HardwareDecodeSuccessful() const { return false; } + /// Return the type name of the class virtual std::string Name() = 0; diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 3cf0bd67..8a933e8a 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,7 @@ using namespace openshot; // Default Constructor for the timeline (which sets the canvas width and height) Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int channels, ChannelLayout channel_layout) : - is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0) + is_open(false), auto_map_clips(true), managed_cache(true), path(""), max_time(0.0), cache_epoch(0), safe_edit_frames_remaining(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -81,7 +82,7 @@ Timeline::Timeline(const ReaderInfo info) : Timeline::Timeline( // Constructor for the timeline (which loads a JSON structure from a file path, and initializes a timeline) Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) : - is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0) { + is_open(false), auto_map_clips(true), managed_cache(true), path(projectPath), max_time(0.0), cache_epoch(0), safe_edit_frames_remaining(0) { // Create CrashHandler and Attach (incase of errors) CrashHandler::Instance(); @@ -102,7 +103,7 @@ Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) // Check if path exists QFileInfo filePath(QString::fromStdString(path)); if (!filePath.exists()) { - throw InvalidFile("File could not be opened.", path); + throw InvalidFile("Timeline project file could not be opened.", path); } // Check OpenShot Install Path exists @@ -585,7 +586,7 @@ std::shared_ptr Timeline::apply_effects(std::shared_ptr frame, int "does_effect_intersect", does_effect_intersect); // Apply the effect to this frame - frame = effect->GetFrame(frame, effect_frame_number); + frame = effect->ProcessFrame(frame, effect_frame_number); } } // end effect loop @@ -632,12 +633,13 @@ std::shared_ptr Timeline::GetOrCreateFrame(std::shared_ptr backgro } // Process a new layer of video or audio -void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, float max_volume) +void Timeline::add_layer(std::shared_ptr new_frame, Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, bool force_safe_composite, float max_volume) { // Create timeline options (with details about this current frame request) TimelineInfoStruct options{}; options.is_top_clip = is_top_clip; options.is_before_clip_keyframes = true; + options.force_safe_composite = force_safe_composite; // Get the clip's frame, composited on top of the current timeline frame std::shared_ptr source_frame; @@ -942,10 +944,13 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Adjust out of bounds frame number if (requested_frame < 1) requested_frame = 1; + const int64_t max_frame = GetMaxFrame(); + const bool past_timeline_end = (max_frame > 0 && requested_frame > max_frame); // Check cache std::shared_ptr frame; - frame = final_cache->GetFrame(requested_frame); + if (!past_timeline_end) + frame = final_cache->GetFrame(requested_frame); if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -962,7 +967,8 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Check cache 2nd time std::shared_ptr frame; - frame = final_cache->GetFrame(requested_frame); + if (!past_timeline_end) + frame = final_cache->GetFrame(requested_frame); if (frame) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -1058,6 +1064,11 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) } // Compose intersecting clips in a single pass + const int safe_remaining = safe_edit_frames_remaining.load(std::memory_order_relaxed); + const bool force_safe_composite = (safe_remaining > 0); + if (force_safe_composite) { + safe_edit_frames_remaining.fetch_sub(1, std::memory_order_relaxed); + } for (const auto& ci : clip_infos) { // Debug output ZmqLogger::Instance()->AppendDebugMethod( @@ -1088,7 +1099,7 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) "clip_frame_number", clip_frame_number); // Add clip's frame as layer - add_layer(new_frame, ci.clip, clip_frame_number, is_top_clip, max_volume_sum); + add_layer(new_frame, ci.clip, clip_frame_number, is_top_clip, force_safe_composite, max_volume_sum); } else { // Debug output @@ -1110,9 +1121,9 @@ std::shared_ptr Timeline::GetFrame(int64_t requested_frame) // Set frame # on mapped frame new_frame->SetFrameNumber(requested_frame); - // Add final frame to cache - final_cache->Add(new_frame); - + // Add final frame to cache (only for valid timeline range) + if (!past_timeline_end) + final_cache->Add(new_frame); // Return frame (or blank frame) return new_frame; } @@ -1344,6 +1355,9 @@ void Timeline::SetJsonValue(const Json::Value root) { // Re-open if needed if (was_open) Open(); + + // Timeline content changed: notify cache clients to rescan active window. + BumpCacheEpoch(); } // Apply a special formatted JSON object, which represents a change to the timeline (insert, update, delete) @@ -1374,6 +1388,13 @@ void Timeline::ApplyJsonDiff(std::string value) { apply_json_to_timeline(change); } + + // Timeline content changed: notify cache clients to rescan active window. + if (!root.empty()) { + // After edits, force safe composition for a short window. + safe_edit_frames_remaining.store(240, std::memory_order_relaxed); + BumpCacheEpoch(); + } } catch (const std::exception& e) { @@ -1382,6 +1403,10 @@ void Timeline::ApplyJsonDiff(std::string value) { } } +void Timeline::BumpCacheEpoch() { + cache_epoch.fetch_add(1, std::memory_order_relaxed); +} + // Apply JSON diff to clips void Timeline::apply_json_to_clips(Json::Value change) { @@ -1434,6 +1459,11 @@ void Timeline::apply_json_to_clips(Json::Value change) { // Apply the change to the effect directly apply_json_to_effects(change, e); + // Effect-only diffs must clear the owning clip cache. + if (existing_clip->GetCache()) { + existing_clip->GetCache()->Clear(); + } + // Calculate start and end frames that this impacts, and remove those frames from the cache int64_t new_starting_frame = (existing_clip->Position() * info.fps.ToDouble()) + 1; int64_t new_ending_frame = ((existing_clip->Position() + existing_clip->Duration()) * info.fps.ToDouble()) + 1; @@ -1485,12 +1515,6 @@ void Timeline::apply_json_to_clips(Json::Value change) { final_cache->Remove(old_starting_frame - 8, old_ending_frame + 8); final_cache->Remove(new_starting_frame - 8, new_ending_frame + 8); - // Remove cache on clip's Reader (if found) - if (existing_clip->Reader() && existing_clip->Reader()->GetCache()) { - existing_clip->Reader()->GetCache()->Remove(old_starting_frame - 8, old_ending_frame + 8); - existing_clip->Reader()->GetCache()->Remove(new_starting_frame - 8, new_ending_frame + 8); - } - // Apply framemapper (or update existing framemapper) if (auto_map_clips) { apply_mapper_to_clip(existing_clip); @@ -1785,6 +1809,9 @@ void Timeline::ClearAllCache(bool deep) { } catch (const ReaderClosed & e) { // ... } + + // Cache content changed: notify cache clients to rebuild their window baseline. + BumpCacheEpoch(); } // Set Max Image Size (used for performance optimization). Convenience function for setting diff --git a/src/Timeline.h b/src/Timeline.h index 83db07ec..d0eb2012 100644 --- a/src/Timeline.h +++ b/src/Timeline.h @@ -15,11 +15,9 @@ #include #include -#include #include -#include -#include -#include +#include +#include #include "TimelineBase.h" #include "ReaderBase.h" @@ -167,11 +165,13 @@ namespace openshot { std::string path; ///< Optional path of loaded UTF-8 OpenShot JSON project file double max_time; ///> The max duration (in seconds) of the timeline, based on the furthest clip (right edge) double min_time; ///> The min duration (in seconds) of the timeline, based on the position of the first clip (left edge) + std::atomic cache_epoch; ///< Cache invalidation epoch for external observers. + std::atomic safe_edit_frames_remaining; ///< Remaining composed frames forced into safe edit composition. std::map> tracked_objects; ///< map of TrackedObjectBBoxes and their IDs /// Process a new layer of video or audio - void add_layer(std::shared_ptr new_frame, openshot::Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, float max_volume); + void add_layer(std::shared_ptr new_frame, openshot::Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, bool force_safe_composite, float max_volume); /// Apply a FrameMapper to a clip which matches the settings of this timeline void apply_mapper_to_clip(openshot::Clip* clip); @@ -211,6 +211,9 @@ namespace openshot { /// Update the list of 'opened' clips void update_open_clips(openshot::Clip *clip, bool does_clip_intersect); + /// Increment the cache invalidation epoch. + void BumpCacheEpoch(); + public: /// @brief Constructor for the timeline (which configures the default frame properties) @@ -313,6 +316,10 @@ namespace openshot { /// of this cache object though (Timeline will not delete it for you). void SetCache(openshot::CacheBase* new_cache); + /// Return the current cache invalidation epoch. + uint64_t CacheEpoch() const { return cache_epoch.load(std::memory_order_relaxed); }; + int SafeEditFramesRemaining() const { return safe_edit_frames_remaining.load(std::memory_order_relaxed); }; + /// Get an openshot::Frame object for a specific frame number of this timeline. /// /// @returns The requested frame (containing the image) diff --git a/src/TimelineBase.h b/src/TimelineBase.h index 46194c50..ee5900f7 100644 --- a/src/TimelineBase.h +++ b/src/TimelineBase.h @@ -33,6 +33,7 @@ namespace openshot { { bool is_top_clip; ///< Is clip on top (if overlapping another clip) bool is_before_clip_keyframes; ///< Is this before clip keyframes are applied + bool force_safe_composite; ///< If true, avoid mutating cached clip images during composition }; /** diff --git a/src/effects/Blur.cpp b/src/effects/Blur.cpp index d1ffe053..2709f4db 100644 --- a/src/effects/Blur.cpp +++ b/src/effects/Blur.cpp @@ -16,7 +16,8 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Blur::Blur() : horizontal_radius(6.0), vertical_radius(6.0), sigma(3.0), iterations(3.0) { +Blur::Blur() : horizontal_radius(6.0), vertical_radius(6.0), sigma(3.0), iterations(3.0), + mask_mode(BLUR_MASK_POST_BLEND) { // Init effect properties init_effect_details(); } @@ -24,7 +25,7 @@ Blur::Blur() : horizontal_radius(6.0), vertical_radius(6.0), sigma(3.0), iterati // Default constructor Blur::Blur(Keyframe new_horizontal_radius, Keyframe new_vertical_radius, Keyframe new_sigma, Keyframe new_iterations) : horizontal_radius(new_horizontal_radius), vertical_radius(new_vertical_radius), - sigma(new_sigma), iterations(new_iterations) + sigma(new_sigma), iterations(new_iterations), mask_mode(BLUR_MASK_POST_BLEND) { // Init effect properties init_effect_details(); @@ -56,6 +57,7 @@ std::shared_ptr Blur::GetFrame(std::shared_ptr int vertical_radius_value = vertical_radius.GetValue(frame_number); float sigma_value = sigma.GetValue(frame_number); int iteration_value = iterations.GetInt(frame_number); + (void) sigma_value; int w = frame_image->width(); int h = frame_image->height(); @@ -90,6 +92,45 @@ std::shared_ptr Blur::GetFrame(std::shared_ptr return frame; } +bool Blur::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == BLUR_MASK_DRIVE_AMOUNT; +} + +void Blur::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + // Use a non-linear response curve for custom blur drive mode. + factor = factor * factor; + const float inverse = 1.0f - factor; + + // Drive blur amount with the grayscale mask while preserving source alpha. + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Credit: http://blog.ivank.net/fastest-gaussian-blur.html (MIT License) // Modified to process all four channels in a pixel array void Blur::boxBlurH(unsigned char *scl, unsigned char *tcl, int w, int h, int r) { @@ -168,6 +209,7 @@ Json::Value Blur::JsonValue() const { root["vertical_radius"] = vertical_radius.JsonValue(); root["sigma"] = sigma.JsonValue(); root["iterations"] = iterations.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -205,6 +247,8 @@ void Blur::SetJsonValue(const Json::Value root) { sigma.SetJsonValue(root["sigma"]); if (!root["iterations"].isNull()) iterations.SetJsonValue(root["iterations"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -218,6 +262,9 @@ std::string Blur::PropertiesJSON(int64_t requested_frame) const { root["vertical_radius"] = add_property_json("Vertical Radius", vertical_radius.GetValue(requested_frame), "float", "", &vertical_radius, 0, 100, false, requested_frame); root["sigma"] = add_property_json("Sigma", sigma.GetValue(requested_frame), "float", "", &sigma, 0, 100, false, requested_frame); root["iterations"] = add_property_json("Iterations", iterations.GetValue(requested_frame), "float", "", &iterations, 0, 100, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", BLUR_MASK_POST_BLEND, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", BLUR_MASK_DRIVE_AMOUNT, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Blur.h b/src/effects/Blur.h index fbf9da25..7cf80bfc 100644 --- a/src/effects/Blur.h +++ b/src/effects/Blur.h @@ -24,6 +24,10 @@ namespace openshot { + enum BlurMaskMode { + BLUR_MASK_POST_BLEND = 0, + BLUR_MASK_DRIVE_AMOUNT = 1 + }; /** * @brief This class adjusts the blur of an image, and can be animated @@ -43,12 +47,17 @@ namespace openshot void boxBlurH(unsigned char *scl, unsigned char *tcl, int w, int h, int r); void boxBlurT(unsigned char *scl, unsigned char *tcl, int w, int h, int r); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; public: Keyframe horizontal_radius; ///< Horizontal blur radius keyframe. The size of the horizontal blur operation in pixels. Keyframe vertical_radius; ///< Vertical blur radius keyframe. The size of the vertical blur operation in pixels. Keyframe sigma; ///< Sigma keyframe. The amount of spread in the blur operation. Should be larger than radius. Keyframe iterations; ///< Iterations keyframe. The # of blur iterations per pixel. 3 iterations = Gaussian. + int mask_mode; ///< How to apply common masks to blur (post-blend or drive-amount). /// Blank constructor, useful when using Json to load the effect properties Blur(); diff --git a/src/effects/Brightness.cpp b/src/effects/Brightness.cpp index 5fdf7f84..59d19cf9 100644 --- a/src/effects/Brightness.cpp +++ b/src/effects/Brightness.cpp @@ -12,17 +12,19 @@ #include "Brightness.h" #include "Exceptions.h" +#include using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Brightness::Brightness() : brightness(0.0), contrast(3.0) { +Brightness::Brightness() : brightness(0.0), contrast(3.0), mask_mode(BRIGHTNESS_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); } // Default constructor -Brightness::Brightness(Keyframe new_brightness, Keyframe new_contrast) : brightness(new_brightness), contrast(new_contrast) +Brightness::Brightness(Keyframe new_brightness, Keyframe new_contrast) : + brightness(new_brightness), contrast(new_contrast), mask_mode(BRIGHTNESS_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); @@ -48,50 +50,138 @@ std::shared_ptr Brightness::GetFrame(std::shared_ptr frame_image = frame->GetImage(); + if (!frame_image) + return frame; // Get keyframe values for this frame - float brightness_value = brightness.GetValue(frame_number); - float contrast_value = contrast.GetValue(frame_number); + const float brightness_value = brightness.GetValue(frame_number); + const float contrast_value = contrast.GetValue(frame_number); + const float contrast_factor = (259.0f * (contrast_value + 255.0f)) / (255.0f * (259.0f - contrast_value)); + const int brightness_offset_i = static_cast(255.0f * brightness_value); // Loop through pixels - unsigned char *pixels = (unsigned char *) frame_image->bits(); - int pixel_count = frame_image->width() * frame_image->height(); + unsigned char *pixels = reinterpret_cast(frame_image->bits()); + const int pixel_count = frame_image->width() * frame_image->height(); + // LUT for undoing premultiplication without a per-pixel divide. + static const std::array inv_alpha = [] { + std::array lut{}; + lut[0] = 0.0f; + for (int i = 1; i < 256; ++i) + lut[i] = 255.0f / static_cast(i); + return lut; + }(); + const auto clamp_u8 = [](int value) -> unsigned char { + if (value < 0) return 0; + if (value > 255) return 255; + return static_cast(value); + }; + const auto clamp_i = [](int value) -> int { + if (value < 0) return 0; + if (value > 255) return 255; + return value; + }; - #pragma omp parallel for + const auto adjust_contrast_and_brightness = [&](int &R, int &G, int &B) { + R = clamp_u8(clamp_i(static_cast((contrast_factor * (R - 128)) + 128.0f)) + brightness_offset_i); + G = clamp_u8(clamp_i(static_cast((contrast_factor * (G - 128)) + 128.0f)) + brightness_offset_i); + B = clamp_u8(clamp_i(static_cast((contrast_factor * (B - 128)) + 128.0f)) + brightness_offset_i); + }; + + #pragma omp parallel for if(pixel_count >= 16384) schedule(static) for (int pixel = 0; pixel < pixel_count; ++pixel) { - // Compute contrast adjustment factor - float factor = (259 * (contrast_value + 255)) / (255 * (259 - contrast_value)); + const int idx = pixel * 4; - // Calculate alpha % (to be used for removing pre-multiplied alpha value) - int A = pixels[pixel * 4 + 3]; - float alpha_percent = A / 255.0; + // Split hot paths by alpha to avoid unnecessary premultiply/unpremultiply work. + const int A = pixels[idx + 3]; + if (A <= 0) + continue; + int R = 0; + int G = 0; + int B = 0; + if (A == 255) { + R = pixels[idx + 0]; + G = pixels[idx + 1]; + B = pixels[idx + 2]; + adjust_contrast_and_brightness(R, G, B); + pixels[idx + 0] = static_cast(R); + pixels[idx + 1] = static_cast(G); + pixels[idx + 2] = static_cast(B); + } else { + const float alpha_percent = static_cast(A) * (1.0f / 255.0f); + const float inv_alpha_percent = inv_alpha[A]; - // Get RGB values, and remove pre-multiplied alpha - unsigned char R = pixels[pixel * 4 + 0] / alpha_percent; - unsigned char G = pixels[pixel * 4 + 1] / alpha_percent; - unsigned char B = pixels[pixel * 4 + 2] / alpha_percent; + // Get RGB values, and remove pre-multiplied alpha + R = static_cast(pixels[idx + 0] * inv_alpha_percent); + G = static_cast(pixels[idx + 1] * inv_alpha_percent); + B = static_cast(pixels[idx + 2] * inv_alpha_percent); + adjust_contrast_and_brightness(R, G, B); - // Apply constrained contrast adjustment - R = constrain((factor * (R - 128)) + 128); - G = constrain((factor * (G - 128)) + 128); - B = constrain((factor * (B - 128)) + 128); - - // Adjust brightness and write constrained values back to image - pixels[pixel * 4 + 0] = constrain(R + (255 * brightness_value)); - pixels[pixel * 4 + 1] = constrain(G + (255 * brightness_value)); - pixels[pixel * 4 + 2] = constrain(B + (255 * brightness_value)); - - // Pre-multiply the alpha back into the color channels - pixels[pixel * 4 + 0] *= alpha_percent; - pixels[pixel * 4 + 1] *= alpha_percent; - pixels[pixel * 4 + 2] *= alpha_percent; + // Pre-multiply alpha back into color channels + pixels[idx + 0] = static_cast(R * alpha_percent); + pixels[idx + 1] = static_cast(G * alpha_percent); + pixels[idx + 2] = static_cast(B * alpha_percent); + } } // return the modified frame return frame; } +bool Brightness::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == BRIGHTNESS_MASK_VARY_STRENGTH; +} + +void Brightness::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + if (mask_invert) { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } + } else { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } + } +} + // Generate JSON string of this object std::string Brightness::Json() const { @@ -107,6 +197,7 @@ Json::Value Brightness::JsonValue() const { root["type"] = info.class_name; root["brightness"] = brightness.JsonValue(); root["contrast"] = contrast.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -140,6 +231,8 @@ void Brightness::SetJsonValue(const Json::Value root) { brightness.SetJsonValue(root["brightness"]); if (!root["contrast"].isNull()) contrast.SetJsonValue(root["contrast"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -151,6 +244,9 @@ std::string Brightness::PropertiesJSON(int64_t requested_frame) const { // Keyframes root["brightness"] = add_property_json("Brightness", brightness.GetValue(requested_frame), "float", "", &brightness, -1.0, 1.0, false, requested_frame); root["contrast"] = add_property_json("Contrast", contrast.GetValue(requested_frame), "float", "", &contrast, -128, 128.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", BRIGHTNESS_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", BRIGHTNESS_MASK_VARY_STRENGTH, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Brightness.h b/src/effects/Brightness.h index d63dc2fc..f58bd0b2 100644 --- a/src/effects/Brightness.h +++ b/src/effects/Brightness.h @@ -24,6 +24,10 @@ namespace openshot { + enum BrightnessMaskMode { + BRIGHTNESS_MASK_LIMIT_TO_AREA = 0, + BRIGHTNESS_MASK_VARY_STRENGTH = 1 + }; /** * @brief This class adjusts the brightness and contrast of an image, and can be animated @@ -37,9 +41,15 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; + public: Keyframe brightness; ///< Brightness keyframe. A constant value here will prevent animation. Keyframe contrast; ///< Contrast keyframe. + int mask_mode; ///< Mask behavior mode for this effect. /// Blank constructor, useful when using Json to load the effect properties Brightness(); diff --git a/src/effects/ChromaKey.cpp b/src/effects/ChromaKey.cpp index 41421eac..0e54cd3a 100644 --- a/src/effects/ChromaKey.cpp +++ b/src/effects/ChromaKey.cpp @@ -16,13 +16,15 @@ #if USE_BABL #include #endif +#include +#include #include #include using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -ChromaKey::ChromaKey() : fuzz(5.0), halo(0), method(CHROMAKEY_BASIC) { +ChromaKey::ChromaKey() : fuzz(20.0), halo(10.0), method(CHROMAKEY_BASIC_SOFT) { // Init default color color = Color(); @@ -114,7 +116,8 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptr CHROMAKEY_BASIC && method <= CHROMAKEY_LAST_METHOD) + if (method != CHROMAKEY_BASIC && method != CHROMAKEY_BASIC_SOFT + && method <= CHROMAKEY_LAST_METHOD) { static bool need_init = true; @@ -282,33 +285,56 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptrscanLine(y); - - for (int x = 0; x < width; ++x, pixel += 4, pc += 3) + case CHROMAKEY_YCBCR: { - int db = (int) pc[1] - mask.u[1]; - int dr = (int) pc[2] - mask.u[2]; - float tmp = sqrt(db * db + dr * dr); + // sqrt(db^2 + dr^2), with db/dr in [-255, 255] + static const std::array sqrt_lut = [] { + std::array lut{}; + for (int i = 0; i <= 130050; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); - if (tmp <= threshold) - { - pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; - } - else if (tmp <= threshold + halothreshold) - { - float alphamult = (tmp - threshold) / halothreshold; + const int threshold_sq = threshold * threshold; + const int halo_upper = threshold + halothreshold; + const int halo_upper_sq = halo_upper * halo_upper; + const float inv_halo = (halothreshold > 0) ? (1.0f / halothreshold) : 0.0f; + const unsigned char key_cb = mask.u[1]; + const unsigned char key_cr = mask.u[2]; + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); - pixel[0] *= alphamult; - pixel[1] *= alphamult; - pixel[2] *= alphamult; - pixel[3] *= alphamult; + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char *pixel = img_bits + (y * img_bpl); + const unsigned char *pc_row = pixelbuf.data() + (y * rowwidth); + + for (int x = 0; x < width; ++x, pixel += 4) + { + const unsigned char *pc_px = pc_row + (x * 3); + int db = static_cast(pc_px[1]) - key_cb; + int dr = static_cast(pc_px[2]) - key_cr; + int dist_sq = db * db + dr * dr; + + if (dist_sq <= threshold_sq) + { + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + else if (halothreshold > 0 && dist_sq <= halo_upper_sq) + { + float tmp = sqrt_lut[dist_sq]; + float alphamult = (tmp - threshold) * inv_halo; + + pixel[0] *= alphamult; + pixel[1] *= alphamult; + pixel[2] *= alphamult; + pixel[3] *= alphamult; + } + } } } - } - break; + break; case CHROMAKEY_CIE_LCH_L: for (int y = 0; y < height; ++y) @@ -485,29 +511,121 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptrscanLine(y); + // Metric upper bound for Color::GetDistance internal term is below 589825. + static const std::array dist_lut = [] { + std::array lut{}; + for (int i = 0; i <= 589825; ++i) + lut[i] = static_cast(std::sqrt(static_cast(i))); + return lut; + }(); + static const std::array inv_alpha = [] { + std::array lut{}; + lut[0] = 0.0f; + for (int i = 1; i < 256; ++i) + lut[i] = 255.0f / static_cast(i); + return lut; + }(); - for (int x = 0; x < width; ++x, pixel += 4) - { - float A = pixel[3]; - unsigned char R = (pixel[0] / A) * 255.0; - unsigned char G = (pixel[1] / A) * 255.0; - unsigned char B = (pixel[2] / A) * 255.0; + if (method == CHROMAKEY_BASIC) { + // Legacy BASIC behavior (hard threshold, no halo), optimized with OpenMP. + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); - // Get distance between mask color and pixel color - long distance = Color::GetDistance((long)R, (long)G, (long)B, mask_R, mask_G, mask_B); + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char * pixel = img_bits + (y * img_bpl); + for (int x = 0; x < width; ++x, pixel += 4) + { + const int A = pixel[3]; + if (A <= 0) + continue; - if (distance <= threshold) { - // MATCHED - Make pixel transparent - // Due to premultiplied alpha, we must also zero out - // the individual color channels (or else artifacts are left behind) - pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + // Preserve legacy 8-bit conversion behavior by casting to unsigned char. + unsigned char R = 0; + unsigned char G = 0; + unsigned char B = 0; + if (A == 255) { + R = pixel[0]; + G = pixel[1]; + B = pixel[2]; + } else { + const float inv_a = inv_alpha[A]; + R = static_cast(pixel[0] * inv_a); + G = static_cast(pixel[1] * inv_a); + B = static_cast(pixel[2] * inv_a); + } + + const int rmean = (static_cast(R) + static_cast(mask_R)) / 2; + const int r = static_cast(R) - static_cast(mask_R); + const int g = static_cast(G) - static_cast(mask_G); + const int b = static_cast(B) - static_cast(mask_B); + const int dist_sq = (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8); + const int distance = dist_lut[dist_sq]; + + if (distance <= threshold) + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + } + } else { + // BASIC_SOFT: same metric, optional halo feathering, optimized + OpenMP. + static const std::array dist_f_lut = [] { + std::array lut{}; + for (int i = 0; i <= 589825; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); + const int halo_upper = threshold + halothreshold; + const int halo_upper_sq = halo_upper * halo_upper; + const float inv_halo = (halothreshold > 0) ? (1.0f / halothreshold) : 0.0f; + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); + + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char * pixel = img_bits + (y * img_bpl); + for (int x = 0; x < width; ++x, pixel += 4) + { + const int A = pixel[3]; + if (A <= 0) + continue; + + int R = 0; + int G = 0; + int B = 0; + if (A == 255) { + R = pixel[0]; + G = pixel[1]; + B = pixel[2]; + } else { + const float inv_a = inv_alpha[A]; + R = std::clamp(static_cast(pixel[0] * inv_a), 0, 255); + G = std::clamp(static_cast(pixel[1] * inv_a), 0, 255); + B = std::clamp(static_cast(pixel[2] * inv_a), 0, 255); + } + + const int rmean = (R + static_cast(mask_R)) / 2; + const int r = R - static_cast(mask_R); + const int g = G - static_cast(mask_G); + const int b = B - static_cast(mask_B); + const int dist_sq = (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8); + + const int distance = dist_lut[dist_sq]; + if (distance <= threshold) { + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + else if (halothreshold > 0 && dist_sq <= halo_upper_sq) { + const float distance_f = dist_f_lut[dist_sq]; + const float alphamult = (distance_f - threshold) * inv_halo; + pixel[0] = static_cast(pixel[0] * alphamult); + pixel[1] = static_cast(pixel[1] * alphamult); + pixel[2] = static_cast(pixel[2] * alphamult); + pixel[3] = static_cast(pixel[3] * alphamult); + } + } } } - } // return the modified frame return frame; @@ -584,6 +702,7 @@ std::string ChromaKey::PropertiesJSON(int64_t requested_frame) const { root["halo"] = add_property_json("Halo", halo.GetValue(requested_frame), "float", "", &halo, 0, 125, false, requested_frame); root["keymethod"] = add_property_json("Key Method", method, "int", "", NULL, 0, CHROMAKEY_LAST_METHOD, false, requested_frame); root["keymethod"]["choices"].append(add_property_choice_json("Basic keying", 0, method)); + root["keymethod"]["choices"].append(add_property_choice_json("Basic keying (soft)", CHROMAKEY_BASIC_SOFT, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSV/HSL hue", 1, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSV saturation", 2, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSL saturation", 3, method)); diff --git a/src/effects/ChromaKey.h b/src/effects/ChromaKey.h index 2a642988..b5dca0a0 100644 --- a/src/effects/ChromaKey.h +++ b/src/effects/ChromaKey.h @@ -56,13 +56,14 @@ namespace openshot /// /// The keying method determines the algorithm to use to determine the distance /// between the key color and the pixel color. The default keying method, - /// CHROMAKEY_BASIC, treates each of the R,G,B values as a vector and calculates + /// CHROMAKEY_BASIC_SOFT, uses the BASIC RGB-distance metric with optional halo feathering. + /// CHROMAKEY_BASIC treats each of the R,G,B values as a vector and calculates /// the length of the difference between those vectors. /// /// Pixels that are less than "fuzz" distance from the key color are eliminated /// by setting their alpha values to zero. /// - /// If halo is non-zero, pixels that are withing the halo distance of the fuzz + /// If halo is non-zero, pixels that are within the halo distance of the fuzz /// distance are given an alpha value that increases with the distance from the /// fuzz boundary. /// @@ -76,7 +77,7 @@ namespace openshot /// @param fuzz The fuzz factor (or threshold) /// @param halo The additional threshold for halo elimination. /// @param method The keying method - ChromaKey(Color color, Keyframe fuzz, Keyframe halo = 0.0, ChromaKeyMethod method = CHROMAKEY_BASIC); + ChromaKey(Color color, Keyframe fuzz, Keyframe halo = 0.0, ChromaKeyMethod method = CHROMAKEY_BASIC_SOFT); /// @brief This method is required for all derived classes of ClipBase, and returns a /// new openshot::Frame object. All Clip keyframes and effects are resolved into diff --git a/src/effects/Hue.cpp b/src/effects/Hue.cpp index 12a64a39..a897670d 100644 --- a/src/effects/Hue.cpp +++ b/src/effects/Hue.cpp @@ -22,7 +22,7 @@ Hue::Hue() : Hue(0.0) { } // Default constructor -Hue::Hue(Keyframe hue): hue(hue) +Hue::Hue(Keyframe hue): hue(hue), mask_mode(HUE_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); @@ -93,6 +93,43 @@ std::shared_ptr Hue::GetFrame(std::shared_ptr return frame; } +bool Hue::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == HUE_MASK_VARY_STRENGTH; +} + +void Hue::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Generate JSON string of this object std::string Hue::Json() const { @@ -107,6 +144,7 @@ Json::Value Hue::JsonValue() const { Json::Value root = EffectBase::JsonValue(); // get parent properties root["type"] = info.class_name; root["hue"] = hue.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -138,6 +176,8 @@ void Hue::SetJsonValue(const Json::Value root) { // Set data from Json (if key is found) if (!root["hue"].isNull()) hue.SetJsonValue(root["hue"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -148,6 +188,9 @@ std::string Hue::PropertiesJSON(int64_t requested_frame) const { // Keyframes root["hue"] = add_property_json("Hue", hue.GetValue(requested_frame), "float", "", &hue, 0.0, 1.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", HUE_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", HUE_MASK_VARY_STRENGTH, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Hue.h b/src/effects/Hue.h index 61599a91..8b95d64a 100644 --- a/src/effects/Hue.h +++ b/src/effects/Hue.h @@ -25,6 +25,10 @@ namespace openshot { + enum HueMaskMode { + HUE_MASK_LIMIT_TO_AREA = 0, + HUE_MASK_VARY_STRENGTH = 1 + }; /** * @brief This class shifts the hue of an image, and can be animated with openshot::Keyframe curves over time. @@ -38,9 +42,14 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; public: Keyframe hue; ///< Shift the hue coordinates (left or right) + int mask_mode; ///< Mask behavior mode for this effect. /// Default constructor, useful when using Json to load the effect properties Hue(); diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index ea5fa887..06842909 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -15,29 +15,27 @@ #include "Exceptions.h" #include "ReaderBase.h" -#include "ChunkReader.h" -#include "FFmpegReader.h" -#include "QtImageReader.h" +#include #include -#ifdef USE_IMAGEMAGICK - #include "ImageReader.h" -#endif - using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Mask::Mask() : reader(NULL), replace_image(false), needs_refresh(true) { +Mask::Mask() : replace_image(false) { // Init effect properties init_effect_details(); } // Default constructor Mask::Mask(ReaderBase *mask_reader, Keyframe mask_brightness, Keyframe mask_contrast) : - reader(mask_reader), brightness(mask_brightness), contrast(mask_contrast), replace_image(false), needs_refresh(true) + brightness(mask_brightness), contrast(mask_contrast), replace_image(false) { // Init effect properties init_effect_details(); + + // Keep ownership local by cloning externally-provided readers. + if (mask_reader) + MaskReader(CreateReaderFromJson(mask_reader->JsonValue())); } // Init effect settings @@ -57,47 +55,20 @@ void Mask::init_effect_details() // This method is required for all derived classes of EffectBase, and returns a // modified openshot::Frame object std::shared_ptr Mask::GetFrame(std::shared_ptr frame, int64_t frame_number) { - // Get the mask image (from the mask reader) + // Get frame image first std::shared_ptr frame_image = frame->GetImage(); - - // Check if mask reader is open - #pragma omp critical (open_mask_reader) - { - if (reader && !reader->IsOpen()) - reader->Open(); - } - - // No reader (bail on applying the mask) - if (!reader) + if (!frame_image || frame_image->isNull()) return frame; - // Get mask image (if missing or different size than frame image) - #pragma omp critical (open_mask_reader) - { - if (!original_mask || !reader->info.has_single_image || needs_refresh || - (original_mask && original_mask->size() != frame_image->size())) { - - // Only get mask if needed - auto mask_without_sizing = std::make_shared( - *reader->GetFrame(frame_number)->GetImage()); - - // Resize mask image to match frame size - original_mask = std::make_shared( - mask_without_sizing->scaled( - frame_image->width(), frame_image->height(), - Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); - } - } - - // Once we've done the necessary resizing, we no longer need to refresh again - needs_refresh = false; + // No reader (bail on applying the mask) + auto original_mask = ResolveMaskImage(frame_image, frame_number); + if (!original_mask || original_mask->isNull()) + return frame; // Grab raw pointers and dimensions one time unsigned char* pixels = reinterpret_cast(frame_image->bits()); unsigned char* mask_pixels = reinterpret_cast(original_mask->bits()); - int width = original_mask->width(); - int height = original_mask->height(); - int num_pixels = width * height; // total pixel count + const int num_pixels = original_mask->width() * original_mask->height(); // Evaluate brightness and contrast keyframes just once double contrast_value = contrast.GetValue(frame_number); @@ -105,47 +76,88 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr int brightness_adj = static_cast(255 * brightness_value); float contrast_factor = 20.0f / std::max(0.00001f, 20.0f - static_cast(contrast_value)); + const bool output_mask = replace_image; + const auto clamp_u8 = [](int value) -> unsigned char { + if (value < 0) return 0; + if (value > 255) return 255; + return static_cast(value); + }; + // Precompute gray->adjusted-gray mapping for this frame's brightness/contrast. + std::array adjusted_gray{}; + for (int gray = 0; gray < 256; ++gray) { + const int adjusted = static_cast(contrast_factor * ((gray + brightness_adj) - 128) + 128); + adjusted_gray[gray] = clamp_u8(adjusted); + } + // 8-bit multiply lookup for premultiplied alpha channel scaling. + static const std::array, 256> mul_lut = [] { + std::array, 256> lut{}; + for (int alpha = 0; alpha < 256; ++alpha) { + for (int value = 0; value < 256; ++value) { + lut[alpha][value] = static_cast((value * alpha) / 255); + } + } + return lut; + }(); - // Iterate over every pixel in parallel -#pragma omp parallel for schedule(static) - for (int i = 0; i < num_pixels; ++i) - { - int idx = i * 4; + // Separate loops keep the hot path branch-free per pixel. + if (output_mask) { + #pragma omp parallel for if(num_pixels >= 16384) schedule(static) + for (int i = 0; i < num_pixels; ++i) { + const int idx = i * 4; + const int R = mask_pixels[idx + 0]; + const int G = mask_pixels[idx + 1]; + const int B = mask_pixels[idx + 2]; + const int A = mask_pixels[idx + 3]; - int R = mask_pixels[idx + 0]; - int G = mask_pixels[idx + 1]; - int B = mask_pixels[idx + 2]; - int A = mask_pixels[idx + 3]; - - // Compute base gray, then apply brightness + contrast - int gray = qGray(R, G, B); - gray += brightness_adj; - gray = static_cast(contrast_factor * (gray - 128) + 128); - - // Clamp (A - gray) into [0, 255] - int diff = A - gray; - if (diff < 0) diff = 0; - else if (diff > 255) diff = 255; - - // Calculate the % change in alpha - float alpha_percent = static_cast(diff) / 255.0f; - - // Set the alpha channel to the gray value - if (replace_image) { - // Replace frame pixels with gray value (including alpha channel) - auto new_val = static_cast(diff); + const int gray = ((R * 11) + (G * 16) + (B * 5)) >> 5; + const int diff = A - adjusted_gray[gray]; + const unsigned char new_val = clamp_u8(diff); pixels[idx + 0] = new_val; pixels[idx + 1] = new_val; pixels[idx + 2] = new_val; pixels[idx + 3] = new_val; - } else { - // Premultiplied RGBA → multiply each channel by alpha_percent - pixels[idx + 0] = static_cast(pixels[idx + 0] * alpha_percent); - pixels[idx + 1] = static_cast(pixels[idx + 1] * alpha_percent); - pixels[idx + 2] = static_cast(pixels[idx + 2] * alpha_percent); - pixels[idx + 3] = static_cast(pixels[idx + 3] * alpha_percent); } + } else if (mask_invert) { + #pragma omp parallel for if(num_pixels >= 16384) schedule(static) + for (int i = 0; i < num_pixels; ++i) { + const int idx = i * 4; + const int R = mask_pixels[idx + 0]; + const int G = mask_pixels[idx + 1]; + const int B = mask_pixels[idx + 2]; + const int A = mask_pixels[idx + 3]; + const int gray = ((R * 11) + (G * 16) + (B * 5)) >> 5; + int alpha = A - adjusted_gray[gray]; + if (alpha < 0) alpha = 0; + else if (alpha > 255) alpha = 255; + alpha = 255 - alpha; + + // Premultiplied RGBA → multiply each channel by alpha + pixels[idx + 0] = mul_lut[alpha][pixels[idx + 0]]; + pixels[idx + 1] = mul_lut[alpha][pixels[idx + 1]]; + pixels[idx + 2] = mul_lut[alpha][pixels[idx + 2]]; + pixels[idx + 3] = mul_lut[alpha][pixels[idx + 3]]; + } + } else { + #pragma omp parallel for if(num_pixels >= 16384) schedule(static) + for (int i = 0; i < num_pixels; ++i) { + const int idx = i * 4; + const int R = mask_pixels[idx + 0]; + const int G = mask_pixels[idx + 1]; + const int B = mask_pixels[idx + 2]; + const int A = mask_pixels[idx + 3]; + + const int gray = ((R * 11) + (G * 16) + (B * 5)) >> 5; + int alpha = A - adjusted_gray[gray]; + if (alpha < 0) alpha = 0; + else if (alpha > 255) alpha = 255; + + // Premultiplied RGBA → multiply each channel by alpha + pixels[idx + 0] = mul_lut[alpha][pixels[idx + 0]]; + pixels[idx + 1] = mul_lut[alpha][pixels[idx + 1]]; + pixels[idx + 2] = mul_lut[alpha][pixels[idx + 2]]; + pixels[idx + 3] = mul_lut[alpha][pixels[idx + 3]]; + } } // return the modified frame @@ -167,10 +179,6 @@ Json::Value Mask::JsonValue() const { root["type"] = info.class_name; root["brightness"] = brightness.JsonValue(); root["contrast"] = contrast.JsonValue(); - if (reader) - root["reader"] = reader->JsonValue(); - else - root["reader"] = Json::objectValue; root["replace_image"] = replace_image; // return JsonValue @@ -196,68 +204,21 @@ void Mask::SetJson(const std::string value) { // Load Json::Value into this object void Mask::SetJsonValue(const Json::Value root) { + Json::Value normalized_root = root; + // Legacy compatibility: keep accepting "reader" on Mask effects. + if (!normalized_root["reader"].isNull() && normalized_root["mask_reader"].isNull()) + normalized_root["mask_reader"] = normalized_root["reader"]; // Set parent data - EffectBase::SetJsonValue(root); + EffectBase::SetJsonValue(normalized_root); // Set data from Json (if key is found) - if (!root["replace_image"].isNull()) - replace_image = root["replace_image"].asBool(); - if (!root["brightness"].isNull()) - brightness.SetJsonValue(root["brightness"]); - if (!root["contrast"].isNull()) - contrast.SetJsonValue(root["contrast"]); - if (!root["reader"].isNull()) // does Json contain a reader? - { - #pragma omp critical (open_mask_reader) - { - // This reader has changed, so refresh cached assets - needs_refresh = true; - - if (!root["reader"]["type"].isNull()) // does the reader Json contain a 'type'? - { - // Close previous reader (if any) - if (reader) { - // Close and delete existing reader (if any) - reader->Close(); - delete reader; - reader = NULL; - } - - // Create new reader (and load properties) - std::string type = root["reader"]["type"].asString(); - - if (type == "FFmpegReader") { - - // Create new reader - reader = new FFmpegReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); - - #ifdef USE_IMAGEMAGICK - } else if (type == "ImageReader") { - - // Create new reader - reader = new ImageReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); - #endif - - } else if (type == "QtImageReader") { - - // Create new reader - reader = new QtImageReader(root["reader"]["path"].asString()); - reader->SetJsonValue(root["reader"]); - - } else if (type == "ChunkReader") { - - // Create new reader - reader = new ChunkReader(root["reader"]["path"].asString(), (ChunkVersion) root["reader"]["chunk_version"].asInt()); - reader->SetJsonValue(root["reader"]); - - } - } - - } - } + if (!normalized_root["replace_image"].isNull()) + replace_image = normalized_root["replace_image"].asBool(); + if (!normalized_root["brightness"].isNull()) + brightness.SetJsonValue(normalized_root["brightness"]); + if (!normalized_root["contrast"].isNull()) + contrast.SetJsonValue(normalized_root["contrast"]); } @@ -276,11 +237,16 @@ std::string Mask::PropertiesJSON(int64_t requested_frame) const { root["brightness"] = add_property_json("Brightness", brightness.GetValue(requested_frame), "float", "", &brightness, -1.0, 1.0, false, requested_frame); root["contrast"] = add_property_json("Contrast", contrast.GetValue(requested_frame), "float", "", &contrast, 0, 20, false, requested_frame); - if (reader) - root["reader"] = add_property_json("Source", 0.0, "reader", reader->Json(), NULL, 0, 1, false, requested_frame); - else - root["reader"] = add_property_json("Source", 0.0, "reader", "{}", NULL, 0, 1, false, requested_frame); - // Return formatted string return root.toStyledString(); } + +void Mask::Reader(ReaderBase *new_reader) { + if (!new_reader) { + MaskReader(NULL); + return; + } + + // Keep ownership local by cloning externally-provided readers. + MaskReader(CreateReaderFromJson(new_reader->JsonValue())); +} diff --git a/src/effects/Mask.h b/src/effects/Mask.h index 36748dd0..8e9eb88d 100644 --- a/src/effects/Mask.h +++ b/src/effects/Mask.h @@ -36,13 +36,12 @@ namespace openshot class Mask : public EffectBase { private: - ReaderBase *reader; - std::shared_ptr original_mask; - bool needs_refresh; - /// Init effect settings void init_effect_details(); + protected: + bool HandlesMaskInternally() const override { return true; } + public: bool replace_image; ///< Replace the frame image with a grayscale image representing the mask. Great for debugging a mask. Keyframe brightness; ///< Brightness keyframe to control the wipe / mask effect. A constant value here will prevent animation. @@ -90,10 +89,10 @@ namespace openshot std::string PropertiesJSON(int64_t requested_frame) const override; /// Get the reader object of the mask grayscale image - ReaderBase* Reader() { return reader; }; + ReaderBase* Reader() { return MaskReader(); }; /// Set a new reader to be used by the mask effect (grayscale image) - void Reader(ReaderBase *new_reader) { reader = new_reader; }; + void Reader(ReaderBase *new_reader); }; } diff --git a/src/effects/Pixelate.cpp b/src/effects/Pixelate.cpp index 76cc3c39..9c3cf9cc 100644 --- a/src/effects/Pixelate.cpp +++ b/src/effects/Pixelate.cpp @@ -22,14 +22,16 @@ using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Pixelate::Pixelate() : pixelization(0.5), left(0.0), top(0.0), right(0.0), bottom(0.0) { +Pixelate::Pixelate() : pixelization(0.5), left(0.0), top(0.0), right(0.0), bottom(0.0), + mask_mode(PIXELATE_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); } // Default constructor Pixelate::Pixelate(Keyframe pixelization, Keyframe left, Keyframe top, Keyframe right, Keyframe bottom) : - pixelization(pixelization), left(left), top(top), right(right), bottom(bottom) + pixelization(pixelization), left(left), top(top), right(right), bottom(bottom), + mask_mode(PIXELATE_MASK_LIMIT_TO_AREA) { // Init effect properties init_effect_details(); @@ -89,6 +91,43 @@ Pixelate::GetFrame(std::shared_ptr frame, int64_t frame_number) return frame; } +bool Pixelate::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == PIXELATE_MASK_VARY_STRENGTH; +} + +void Pixelate::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Generate JSON string of this object std::string Pixelate::Json() const { @@ -107,6 +146,7 @@ Json::Value Pixelate::JsonValue() const { root["top"] = top.JsonValue(); root["right"] = right.JsonValue(); root["bottom"] = bottom.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -146,6 +186,8 @@ void Pixelate::SetJsonValue(const Json::Value root) { right.SetJsonValue(root["right"]); if (!root["bottom"].isNull()) bottom.SetJsonValue(root["bottom"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -160,6 +202,9 @@ std::string Pixelate::PropertiesJSON(int64_t requested_frame) const { root["top"] = add_property_json("Top Margin", top.GetValue(requested_frame), "float", "", &top, 0.0, 1.0, false, requested_frame); root["right"] = add_property_json("Right Margin", right.GetValue(requested_frame), "float", "", &right, 0.0, 1.0, false, requested_frame); root["bottom"] = add_property_json("Bottom Margin", bottom.GetValue(requested_frame), "float", "", &bottom, 0.0, 1.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", PIXELATE_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", PIXELATE_MASK_VARY_STRENGTH, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Pixelate.h b/src/effects/Pixelate.h index 08daa76d..55186d75 100644 --- a/src/effects/Pixelate.h +++ b/src/effects/Pixelate.h @@ -24,6 +24,10 @@ namespace openshot { + enum PixelateMaskMode { + PIXELATE_MASK_LIMIT_TO_AREA = 0, + PIXELATE_MASK_VARY_STRENGTH = 1 + }; /** * @brief This class pixelates an image, and can be animated with openshot::Keyframe curves over time. @@ -37,6 +41,10 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; public: Keyframe pixelization; ///< Amount of pixelization @@ -44,6 +52,7 @@ namespace openshot Keyframe top; ///< Size of top margin Keyframe right; ///< Size of right margin Keyframe bottom; ///< Size of bottom margin + int mask_mode; ///< Mask behavior mode for this effect. /// Default constructor, useful when using Json to load the effect properties Pixelate(); diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index 2e06db9e..a1028425 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -12,23 +12,85 @@ #include "Saturation.h" #include "Exceptions.h" +#include +#include using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -Saturation::Saturation() : saturation(1.0), saturation_R(1.0), saturation_G(1.0), saturation_B(1.0) { +Saturation::Saturation() : saturation(1.0), saturation_R(1.0), saturation_G(1.0), saturation_B(1.0), + mask_mode(SATURATION_MASK_POST_BLEND) { // Init effect properties init_effect_details(); } // Default constructor Saturation::Saturation(Keyframe saturation, Keyframe saturation_R, Keyframe saturation_G, Keyframe saturation_B) : - saturation(saturation), saturation_R(saturation_R), saturation_G(saturation_G), saturation_B(saturation_B) + saturation(saturation), saturation_R(saturation_R), saturation_G(saturation_G), saturation_B(saturation_B), + mask_mode(SATURATION_MASK_POST_BLEND) { // Init effect properties init_effect_details(); } +bool Saturation::UseCustomMaskBlend(int64_t frame_number) const { + (void) frame_number; + return mask_mode == SATURATION_MASK_DRIVE_AMOUNT; +} + +void Saturation::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const { + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + if (mask_invert) { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + factor = 1.0f - factor; + // Use a non-linear response curve for custom saturation drive mode. + factor = factor * factor; + const float inverse = 1.0f - factor; + + // Drive saturation strength with mask while preserving source alpha. + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } + } else { + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + // Use a non-linear response curve for custom saturation drive mode. + factor = factor * factor; + const float inverse = 1.0f - factor; + + // Drive saturation strength with mask while preserving source alpha. + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } + } +} + // Init effect settings void Saturation::init_effect_details() { @@ -53,94 +115,116 @@ std::shared_ptr Saturation::GetFrame(std::shared_ptrwidth() * frame_image->height(); + const int pixel_count = frame_image->width() * frame_image->height(); // Get keyframe values for this frame - float saturation_value = saturation.GetValue(frame_number); - float saturation_value_R = saturation_R.GetValue(frame_number); - float saturation_value_G = saturation_G.GetValue(frame_number); - float saturation_value_B = saturation_B.GetValue(frame_number); + const float saturation_value = saturation.GetValue(frame_number); + const float saturation_value_R = saturation_R.GetValue(frame_number); + const float saturation_value_G = saturation_G.GetValue(frame_number); + const float saturation_value_B = saturation_B.GetValue(frame_number); // Constants used for color saturation formula - const double pR = .299; - const double pG = .587; - const double pB = .114; + const float pR = 0.299f; + const float pG = 0.587f; + const float pB = 0.114f; + const float sqrt_pR = std::sqrt(pR); + const float sqrt_pG = std::sqrt(pG); + const float sqrt_pB = std::sqrt(pB); + // Rec.601 fixed-point luma weights used in many image/video pipelines. + // This avoids per-pixel sqrt() while keeping output stable. + static const std::array sqrt_lut = [] { + std::array lut{}; + for (int i = 0; i <= 65025; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); // Loop through pixels - unsigned char *pixels = (unsigned char *) frame_image->bits(); + unsigned char *pixels = reinterpret_cast(frame_image->bits()); + // LUT for undoing premultiplication without a per-pixel divide. + static const std::array inv_alpha = [] { + std::array lut{}; + lut[0] = 0.0f; + for (int i = 1; i < 256; ++i) + lut[i] = 255.0f / static_cast(i); + return lut; + }(); + const auto clamp_i = [](int value) -> int { + if (value < 0) return 0; + if (value > 255) return 255; + return value; + }; - #pragma omp parallel for shared (pixels) + const auto apply_saturation = [&](int &R, int &G, int &B) { + // Approximate sqrt(R^2*pR + G^2*pG + B^2*pB) with fixed-point weighted + // intensity and lookup table. 77/150/29 ~= 0.299/0.587/0.114. + const int weighted_sq = (77 * R * R + 150 * G * G + 29 * B * B + 128) >> 8; + const float p = sqrt_lut[weighted_sq]; + + // Adjust common saturation + R = clamp_i(static_cast(p + (R - p) * saturation_value)); + G = clamp_i(static_cast(p + (G - p) * saturation_value)); + B = clamp_i(static_cast(p + (B - p) * saturation_value)); + + // Compute per-channel replacement brightness + const float p_r = R * sqrt_pR; + const float p_g = G * sqrt_pG; + const float p_b = B * sqrt_pB; + + // Adjust channel-separated saturation + const int Rr = static_cast(p_r + (R - p_r) * saturation_value_R); + const int Gr = static_cast(p_r - p_r * saturation_value_R); + const int Br = static_cast(p_r - p_r * saturation_value_R); + + const int Rg = static_cast(p_g - p_g * saturation_value_G); + const int Gg = static_cast(p_g + (G - p_g) * saturation_value_G); + const int Bg = static_cast(p_g - p_g * saturation_value_G); + + const int Rb = static_cast(p_b - p_b * saturation_value_B); + const int Gb = static_cast(p_b - p_b * saturation_value_B); + const int Bb = static_cast(p_b + (B - p_b) * saturation_value_B); + + // Recombine and constrain values + R = clamp_i(Rr + Rg + Rb); + G = clamp_i(Gr + Gg + Gb); + B = clamp_i(Br + Bg + Bb); + }; + + #pragma omp parallel for if(pixel_count >= 16384) schedule(static) shared (pixels) for (int pixel = 0; pixel < pixel_count; ++pixel) { - // Calculate alpha % (to be used for removing pre-multiplied alpha value) - int A = pixels[pixel * 4 + 3]; - float alpha_percent = A / 255.0; + const int idx = pixel * 4; - // Get RGB values, and remove pre-multiplied alpha - int R = pixels[pixel * 4 + 0] / alpha_percent; - int G = pixels[pixel * 4 + 1] / alpha_percent; - int B = pixels[pixel * 4 + 2] / alpha_percent; + // Split hot paths by alpha to avoid unnecessary premultiply/unpremultiply work. + const int A = pixels[idx + 3]; + if (A <= 0) + continue; + int R = 0; + int G = 0; + int B = 0; + if (A == 255) { + R = pixels[idx + 0]; + G = pixels[idx + 1]; + B = pixels[idx + 2]; + apply_saturation(R, G, B); + pixels[idx + 0] = static_cast(R); + pixels[idx + 1] = static_cast(G); + pixels[idx + 2] = static_cast(B); + } else { + const float alpha_percent = static_cast(A) * (1.0f / 255.0f); + const float inv_alpha_percent = inv_alpha[A]; - /* - * Common saturation adjustment - */ + // Get RGB values, and remove pre-multiplied alpha + R = static_cast(pixels[idx + 0] * inv_alpha_percent); + G = static_cast(pixels[idx + 1] * inv_alpha_percent); + B = static_cast(pixels[idx + 2] * inv_alpha_percent); + apply_saturation(R, G, B); - // Calculate the saturation multiplier - double p = sqrt( (R * R * pR) + - (G * G * pG) + - (B * B * pB) ); - - // Adjust the saturation - R = constrain(p + (R - p) * saturation_value); - G = constrain(p + (G - p) * saturation_value); - B = constrain(p + (B - p) * saturation_value); - - /* - * Color-separated saturation adjustment - * - * Splitting each of the three subpixels (R, G and B) into three distincs sub-subpixels (R, G and B in turn) - * which in their optical sum reproduce the original subpixel's color OR produce white light in the brightness - * of the original subpixel (dependening on the color channel's slider value). - */ - - // Compute the brightness ("saturation multiplier") of the replaced subpixels - // Actually mathematical no-ops mostly, verbosity is kept just for clarification - const double p_r = sqrt(R * R * pR); - const double p_g = sqrt(G * G * pG); - const double p_b = sqrt(B * B * pB); - - // Adjust the saturation - const int Rr = p_r + (R - p_r) * saturation_value_R; - const int Gr = p_r + (0 - p_r) * saturation_value_R; - const int Br = p_r + (0 - p_r) * saturation_value_R; - - const int Rg = p_g + (0 - p_g) * saturation_value_G; - const int Gg = p_g + (G - p_g) * saturation_value_G; - const int Bg = p_g + (0 - p_g) * saturation_value_G; - - const int Rb = p_b + (0 - p_b) * saturation_value_B; - const int Gb = p_b + (0 - p_b) * saturation_value_B; - const int Bb = p_b + (B - p_b) * saturation_value_B; - - // Recombine brightness of sub-subpixels (Rx, Gx and Bx) into sub-pixels (R, G and B) again - R = Rr + Rg + Rb; - G = Gr + Gg + Gb; - B = Br + Bg + Bb; - - // Constrain the value from 0 to 255 - R = constrain(R); - G = constrain(G); - B = constrain(B); - - // Set all pixels to new value - pixels[pixel * 4 + 0] = R; - pixels[pixel * 4 + 1] = G; - pixels[pixel * 4 + 2] = B; - - // Pre-multiply the alpha back into the color channels - pixels[pixel * 4 + 0] *= alpha_percent; - pixels[pixel * 4 + 1] *= alpha_percent; - pixels[pixel * 4 + 2] *= alpha_percent; + // Pre-multiply alpha back into color channels + pixels[idx + 0] = static_cast(R * alpha_percent); + pixels[idx + 1] = static_cast(G * alpha_percent); + pixels[idx + 2] = static_cast(B * alpha_percent); + } } // return the modified frame @@ -164,6 +248,7 @@ Json::Value Saturation::JsonValue() const { root["saturation_R"] = saturation_R.JsonValue(); root["saturation_G"] = saturation_G.JsonValue(); root["saturation_B"] = saturation_B.JsonValue(); + root["mask_mode"] = mask_mode; // return JsonValue return root; @@ -201,6 +286,8 @@ void Saturation::SetJsonValue(const Json::Value root) { saturation_G.SetJsonValue(root["saturation_G"]); if (!root["saturation_B"].isNull()) saturation_B.SetJsonValue(root["saturation_B"]); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // Get all properties for a specific frame @@ -214,6 +301,9 @@ std::string Saturation::PropertiesJSON(int64_t requested_frame) const { root["saturation_R"] = add_property_json("Saturation (Red)", saturation_R.GetValue(requested_frame), "float", "", &saturation_R, 0.0, 4.0, false, requested_frame); root["saturation_G"] = add_property_json("Saturation (Green)", saturation_G.GetValue(requested_frame), "float", "", &saturation_G, 0.0, 4.0, false, requested_frame); root["saturation_B"] = add_property_json("Saturation (Blue)", saturation_B.GetValue(requested_frame), "float", "", &saturation_B, 0.0, 4.0, false, requested_frame); + root["mask_mode"] = add_property_json("Mask Mode", mask_mode, "int", "", NULL, 0, 1, false, requested_frame); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", SATURATION_MASK_POST_BLEND, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", SATURATION_MASK_DRIVE_AMOUNT, mask_mode)); // Return formatted string return root.toStyledString(); diff --git a/src/effects/Saturation.h b/src/effects/Saturation.h index b8861961..cea64cdf 100644 --- a/src/effects/Saturation.h +++ b/src/effects/Saturation.h @@ -32,6 +32,10 @@ namespace openshot { + enum SaturationMaskMode { + SATURATION_MASK_POST_BLEND = 0, + SATURATION_MASK_DRIVE_AMOUNT = 1 + }; /** * @brief This class adjusts the saturation of color on a frame's image. @@ -45,11 +49,17 @@ namespace openshot /// Init effect settings void init_effect_details(); + protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; + public: Keyframe saturation; ///< Overall color saturation: 0.0 = greyscale, 1.0 = normal, 2.0 = double saturation Keyframe saturation_R; ///< Red color saturation Keyframe saturation_G; ///< Green color saturation Keyframe saturation_B; ///< Blue color saturation + int mask_mode; ///< How to apply common masks to saturation (post-blend or drive-amount). /// Blank constructor, useful when using Json to load the effect properties Saturation(); diff --git a/src/effects/Sharpen.cpp b/src/effects/Sharpen.cpp index b75faa3f..bc1fb374 100644 --- a/src/effects/Sharpen.cpp +++ b/src/effects/Sharpen.cpp @@ -27,6 +27,7 @@ Sharpen::Sharpen() , threshold(0.0) , mode(0) , channel(1) + , mask_mode(SHARPEN_MASK_LIMIT_TO_AREA) { init_effect_details(); } @@ -38,6 +39,7 @@ Sharpen::Sharpen(Keyframe a, Keyframe r, Keyframe t) , threshold(t) , mode(0) , channel(1) + , mask_mode(SHARPEN_MASK_LIMIT_TO_AREA) { init_effect_details(); } @@ -53,6 +55,45 @@ void Sharpen::init_effect_details() info.has_video = true; } +bool Sharpen::UseCustomMaskBlend(int64_t frame_number) const +{ + (void) frame_number; + return mask_mode == SHARPEN_MASK_VARY_STRENGTH; +} + +void Sharpen::ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const +{ + (void) frame_number; + if (!original_image || !effected_image || !mask_image) + return; + if (original_image->size() != effected_image->size() || effected_image->size() != mask_image->size()) + return; + + unsigned char* original_pixels = reinterpret_cast(original_image->bits()); + unsigned char* effected_pixels = reinterpret_cast(effected_image->bits()); + unsigned char* mask_pixels = reinterpret_cast(mask_image->bits()); + const int pixel_count = effected_image->width() * effected_image->height(); + + #pragma omp parallel for schedule(static) + for (int i = 0; i < pixel_count; ++i) { + const int idx = i * 4; + float factor = static_cast(qGray(mask_pixels[idx], mask_pixels[idx + 1], mask_pixels[idx + 2])) / 255.0f; + if (mask_invert) + factor = 1.0f - factor; + factor = factor * factor; + const float inverse = 1.0f - factor; + + effected_pixels[idx] = static_cast( + (original_pixels[idx] * inverse) + (effected_pixels[idx] * factor)); + effected_pixels[idx + 1] = static_cast( + (original_pixels[idx + 1] * inverse) + (effected_pixels[idx + 1] * factor)); + effected_pixels[idx + 2] = static_cast( + (original_pixels[idx + 2] * inverse) + (effected_pixels[idx + 2] * factor)); + effected_pixels[idx + 3] = original_pixels[idx + 3]; + } +} + // Compute three box sizes to approximate a Gaussian of sigma static void boxes_for_gauss(double sigma, int b[3]) { @@ -346,6 +387,7 @@ Json::Value Sharpen::JsonValue() const root["threshold"] = threshold.JsonValue(); root["mode"] = mode; root["channel"] = channel; + root["mask_mode"] = mask_mode; return root; } @@ -369,6 +411,8 @@ void Sharpen::SetJsonValue(Json::Value root) mode = root["mode"].asInt(); if (!root["channel"].isNull()) channel = root["channel"].asInt(); + if (!root["mask_mode"].isNull()) + mask_mode = root["mask_mode"].asInt(); } // UI property definitions @@ -390,5 +434,9 @@ std::string Sharpen::PropertiesJSON(int64_t t) const root["channel"]["choices"].append(add_property_choice_json("All", 0, channel)); root["channel"]["choices"].append(add_property_choice_json("Luma", 1, channel)); root["channel"]["choices"].append(add_property_choice_json("Chroma", 2, channel)); + root["mask_mode"] = add_property_json( + "Mask Mode", mask_mode, "int", "", nullptr, 0, 1, false, t); + root["mask_mode"]["choices"].append(add_property_choice_json("Limit to Mask", SHARPEN_MASK_LIMIT_TO_AREA, mask_mode)); + root["mask_mode"]["choices"].append(add_property_choice_json("Vary Strength", SHARPEN_MASK_VARY_STRENGTH, mask_mode)); return root.toStyledString(); } diff --git a/src/effects/Sharpen.h b/src/effects/Sharpen.h index 7021a9d1..64929bcd 100644 --- a/src/effects/Sharpen.h +++ b/src/effects/Sharpen.h @@ -22,6 +22,11 @@ namespace openshot { +enum SharpenMaskMode { + SHARPEN_MASK_LIMIT_TO_AREA = 0, + SHARPEN_MASK_VARY_STRENGTH = 1 +}; + /** * @brief This class provides a sharpen effect for video frames. * @@ -33,6 +38,11 @@ private: /// Initialize the effect details void init_effect_details(); +protected: + bool UseCustomMaskBlend(int64_t frame_number) const override; + void ApplyCustomMaskBlend(std::shared_ptr original_image, std::shared_ptr effected_image, + std::shared_ptr mask_image, int64_t frame_number) const override; + public: /// Amount of sharpening to apply (0 to 2) Keyframe amount; @@ -49,6 +59,9 @@ public: /// Channel to apply sharpening to (0 = All, 1 = Luma, 2 = Chroma) int channel; + /// Mask behavior mode for this effect. + int mask_mode; + /// Default constructor Sharpen(); @@ -81,4 +94,4 @@ public: } // namespace openshot -#endif // OPENSHOT_SHARPEN_EFFECT_H \ No newline at end of file +#endif // OPENSHOT_SHARPEN_EFFECT_H diff --git a/tests/Benchmark.cpp b/tests/Benchmark.cpp index 66954257..768fb4fa 100644 --- a/tests/Benchmark.cpp +++ b/tests/Benchmark.cpp @@ -9,6 +9,7 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include +#include #include #include #include @@ -27,6 +28,7 @@ #include "ReaderBase.h" #include "Timeline.h" #include "effects/Brightness.h" +#include "effects/ChromaKey.h" #include "effects/Crop.h" #include "effects/Mask.h" #include "effects/Saturation.h" @@ -35,6 +37,8 @@ using namespace openshot; using namespace std; using Clock = chrono::steady_clock; +using TrialFunc = function; +using Trial = pair; template double time_trial(const string &name, Func func) { auto start = Clock::now(); @@ -53,22 +57,42 @@ void read_forward_backward(ReaderBase &reader) { reader.GetFrame(i); } -int main() { - cout << "Trial,Milliseconds\n"; - double total = 0.0; +int main(int argc, char* argv[]) { const string base = TEST_MEDIA_PATH; const string video = base + "sintel_trailer-720p.mp4"; const string mask_img = base + "mask.png"; const string overlay = base + "front3.png"; + string filter_test; + bool list_only = false; + const int64_t chroma_bench_frames = 500; - total += time_trial("FFmpegReader", [&]() { + for (int i = 1; i < argc; ++i) { + const string arg = argv[i]; + if ((arg == "--test" || arg == "-t") && i + 1 < argc) { + filter_test = argv[++i]; + } else if (arg == "--list" || arg == "-l") { + list_only = true; + } else if (arg == "--help" || arg == "-h") { + cout << "Usage: openshot-benchmark [--test ] [--list]\n"; + return 0; + } else { + cerr << "Unknown argument: " << arg << "\n"; + cerr << "Usage: openshot-benchmark [--test ] [--list]\n"; + return 1; + } + } + + vector trials; + trials.reserve(10); + + trials.emplace_back("FFmpegReader", [&]() { FFmpegReader r(video); r.Open(); read_forward_backward(r); r.Close(); }); - total += time_trial("FFmpegWriter", [&]() { + trials.emplace_back("FFmpegWriter", [&]() { FFmpegReader r(video); r.Open(); FFmpegWriter w("benchmark_output.mp4"); @@ -82,7 +106,7 @@ int main() { r.Close(); }); - total += time_trial("FrameMapper", [&]() { + trials.emplace_back("FrameMapper", [&]() { vector rates = {Fraction(24, 1), Fraction(30, 1), Fraction(60, 1), Fraction(30000, 1001), Fraction(60000, 1001)}; for (auto &fps : rates) { @@ -98,14 +122,14 @@ int main() { } }); - total += time_trial("Clip", [&]() { + trials.emplace_back("Clip", [&]() { Clip c(video); c.Open(); read_forward_backward(c); c.Close(); }); - total += time_trial("Timeline", [&]() { + trials.emplace_back("Timeline", [&]() { Timeline t(1920, 1080, Fraction(24, 1), 44100, 2, LAYOUT_STEREO); Clip video_clip(video); video_clip.Layer(0); @@ -131,7 +155,7 @@ int main() { t.Close(); }); - total += time_trial("Timeline (with transforms)", [&]() { + trials.emplace_back("Timeline (with transforms)", [&]() { Timeline t(1920, 1080, Fraction(24, 1), 44100, 2, LAYOUT_STEREO); Clip video_clip(video); int64_t last = video_clip.Reader()->info.video_length; @@ -165,26 +189,26 @@ int main() { t.Close(); }); - total += time_trial("Effect_Mask", [&]() { - FFmpegReader r(video); - r.Open(); + trials.emplace_back("Effect_Mask", [&]() { + FFmpegReader r(video); + r.Open(); #ifdef USE_IMAGEMAGICK - ImageReader mask_reader(mask_img); + ImageReader mask_reader(mask_img); #else - QtImageReader mask_reader(mask_img); + QtImageReader mask_reader(mask_img); #endif - mask_reader.Open(); - Clip clip(&r); - clip.Open(); - Mask m(&mask_reader, Keyframe(0.0), Keyframe(0.5)); - clip.AddEffect(&m); - read_forward_backward(clip); - mask_reader.Close(); - clip.Close(); - r.Close(); - }); + mask_reader.Open(); + Clip clip(&r); + clip.Open(); + Mask m(&mask_reader, Keyframe(0.0), Keyframe(0.5)); + clip.AddEffect(&m); + read_forward_backward(clip); + mask_reader.Close(); + clip.Close(); + r.Close(); + }); - total += time_trial("Effect_Brightness", [&]() { + trials.emplace_back("Effect_Brightness", [&]() { FFmpegReader r(video); r.Open(); Clip clip(&r); @@ -196,7 +220,7 @@ int main() { r.Close(); }); - total += time_trial("Effect_Crop", [&]() { + trials.emplace_back("Effect_Crop", [&]() { FFmpegReader r(video); r.Open(); Clip clip(&r); @@ -208,7 +232,7 @@ int main() { r.Close(); }); - total += time_trial("Effect_Saturation", [&]() { + trials.emplace_back("Effect_Saturation", [&]() { FFmpegReader r(video); r.Open(); Clip clip(&r); @@ -221,6 +245,62 @@ int main() { r.Close(); }); + trials.emplace_back("Effect_ChromaKey_BASIC", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + // Default/basic chroma key method baseline + ChromaKey key(Color(0, 255, 0, 255), Keyframe(80.0), Keyframe(20.0), CHROMAKEY_BASIC); + clip.AddEffect(&key); + const int64_t bench_frames = std::min(clip.info.video_length, chroma_bench_frames); + for (int64_t i = 1; i <= bench_frames; ++i) + clip.GetFrame(i); + for (int64_t i = bench_frames; i >= 1; --i) + clip.GetFrame(i); + clip.Close(); + r.Close(); + }); + + trials.emplace_back("Effect_ChromaKey_BASIC_SOFT", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + ChromaKey key(Color(0, 255, 0, 255), Keyframe(80.0), Keyframe(20.0), CHROMAKEY_BASIC_SOFT); + clip.AddEffect(&key); + const int64_t bench_frames = std::min(clip.info.video_length, chroma_bench_frames); + for (int64_t i = 1; i <= bench_frames; ++i) + clip.GetFrame(i); + for (int64_t i = bench_frames; i >= 1; --i) + clip.GetFrame(i); + clip.Close(); + r.Close(); + }); + + if (list_only) { + for (const auto& trial : trials) + cout << trial.first << "\n"; + return 0; + } + + cout << "Trial,Milliseconds\n"; + double total = 0.0; + int executed = 0; + for (const auto& trial : trials) { + if (!filter_test.empty() && trial.first != filter_test) + continue; + total += time_trial(trial.first, trial.second); + executed++; + } + + if (!filter_test.empty() && executed == 0) { + cerr << "Unknown test: " << filter_test << "\nAvailable tests:\n"; + for (const auto& trial : trials) + cerr << " " << trial.first << "\n"; + return 2; + } + cout << "Overall," << total << "\n"; return 0; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2e7e72ab..7ad186ea 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -55,6 +55,8 @@ set(OPENSHOT_TESTS Crop LensFlare AnalogTape + EffectMask + Mask Sharpen SphericalEffect WaveEffect @@ -63,6 +65,7 @@ set(OPENSHOT_TESTS # ImageMagick related test files if($CACHE{HAVE_IMAGEMAGICK}) list(APPEND OPENSHOT_TESTS + ImageReader ImageWriter ) endif() diff --git a/tests/ChromaKey.cpp b/tests/ChromaKey.cpp index c478bcba..af6e0827 100644 --- a/tests/ChromaKey.cpp +++ b/tests/ChromaKey.cpp @@ -69,3 +69,66 @@ TEST_CASE( "threshold", "[libopenshot][effect][chromakey]" ) CHECK(pix_e == expected); } +TEST_CASE( "default method is basic soft", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::ChromaKey e; + Json::Value json = e.JsonValue(); + CHECK(json["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} + +TEST_CASE( "basic vs basic soft halo behavior", "[libopenshot][effect][chromakey]" ) +{ + // Pick a green value in the halo band for key=(0,255,0), threshold=5, halo=20. + // For BASIC this should remain unchanged, for BASIC_SOFT it should be partially faded. + auto frame_basic = std::make_shared(1, 320, 180, "#00fa00"); + auto frame_soft = std::make_shared(1, 320, 180, "#00fa00"); + + openshot::Color key(0, 255, 0, 255); + openshot::Keyframe fuzz(5); + openshot::Keyframe halo(20); + + openshot::ChromaKey basic(key, fuzz, halo, CHROMAKEY_BASIC); + openshot::ChromaKey soft(key, fuzz, halo, CHROMAKEY_BASIC_SOFT); + + auto out_basic = basic.GetFrame(frame_basic, 1); + auto out_soft = soft.GetFrame(frame_soft, 1); + + QColor basic_pix = out_basic->GetImage()->pixelColor(10, 10); + QColor soft_pix = out_soft->GetImage()->pixelColor(10, 10); + + CHECK(basic_pix == QColor(0, 250, 0, 255)); + CHECK(soft_pix.alpha() < 255); + CHECK(soft_pix.alpha() > 0); + CHECK(soft_pix != basic_pix); +} + +TEST_CASE( "json roundtrip preserves method", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::Color key(0, 255, 0, 255); + openshot::Keyframe fuzz(5); + openshot::Keyframe halo(20); + openshot::ChromaKey basic(key, fuzz, halo, CHROMAKEY_BASIC); + openshot::ChromaKey soft(key, fuzz, halo, CHROMAKEY_BASIC_SOFT); + + Json::Value basic_json = basic.JsonValue(); + Json::Value soft_json = soft.JsonValue(); + + openshot::ChromaKey basic_loaded; + openshot::ChromaKey soft_loaded; + basic_loaded.SetJsonValue(basic_json); + soft_loaded.SetJsonValue(soft_json); + + CHECK(basic_loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC); + CHECK(soft_loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} + +TEST_CASE( "SetJson string preserves keymethod", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::ChromaKey source(openshot::Color(0, 255, 0, 255), openshot::Keyframe(5), openshot::Keyframe(20), CHROMAKEY_BASIC_SOFT); + const std::string payload = source.Json(); + + openshot::ChromaKey loaded; + loaded.SetJson(payload); + + CHECK(loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} diff --git a/tests/Clip.cpp b/tests/Clip.cpp index 53500d62..e06c34ef 100644 --- a/tests/Clip.cpp +++ b/tests/Clip.cpp @@ -760,6 +760,43 @@ TEST_CASE( "composite_over_opaque_background_blend", "[libopenshot][clip][pr]" ) CHECK(center.blue() == Approx(127).margin(12)); } +TEST_CASE( "cached_frame_not_mutated_by_background_compositing", "[libopenshot][clip][cache]" ) +{ + // Source clip: solid red + openshot::CacheMemory cache; + auto src = std::make_shared(1, 64, 64, "#000000", 0, 2); + src->AddColor(QColor(Qt::red)); + cache.Add(src); + + openshot::DummyReader dummy(openshot::Fraction(30,1), 64, 64, 44100, 2, 1.0, &cache); + dummy.Open(); + + openshot::Clip clip; + clip.Reader(&dummy); + clip.Open(); + clip.display = openshot::FRAME_DISPLAY_NONE; + clip.alpha.AddPoint(1, 0.5); // semi-transparent source to reveal background + + // First composite over blue background (expect purple-ish) + auto bg_blue = std::make_shared(1, 64, 64, "#000000", 0, 2); + bg_blue->AddColor(QColor(Qt::blue)); + auto out_blue = clip.GetFrame(bg_blue, 1); + QColor c1 = out_blue->GetImage()->pixelColor(32, 32); + CHECK(c1.red() == Approx(127).margin(14)); + CHECK(c1.green() == Approx(0).margin(8)); + CHECK(c1.blue() == Approx(127).margin(14)); + + // Second composite of same clip frame over green background should be yellow-ish. + // If cached frame was mutated by the first call, this will incorrectly remain purple-ish. + auto bg_green = std::make_shared(1, 64, 64, "#000000", 0, 2); + bg_green->AddColor(QColor(Qt::green)); + auto out_green = clip.GetFrame(bg_green, 1); + QColor c2 = out_green->GetImage()->pixelColor(32, 32); + CHECK(c2.red() == Approx(127).margin(14)); + CHECK(c2.green() == Approx(127).margin(14)); + CHECK(c2.blue() == Approx(0).margin(8)); +} + TEST_CASE("all_composite_modes_simple_colors", "[libopenshot][clip][composite]") { // Source clip: solid red diff --git a/tests/EffectMask.cpp b/tests/EffectMask.cpp new file mode 100644 index 00000000..7c7c018c --- /dev/null +++ b/tests/EffectMask.cpp @@ -0,0 +1,611 @@ +/** + * @file + * @brief Unit tests for common EffectBase mask dispatch and blur mask modes + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "CacheMemory.h" +#include "Clip.h" +#include "DummyReader.h" +#include "effects/Blur.h" +#include "effects/Brightness.h" +#include "effects/Hue.h" +#include "effects/Pixelate.h" +#include "effects/Saturation.h" +#include "effects/Sharpen.h" +#include "QtImageReader.h" +#include "openshot_catch.h" + +using namespace openshot; + +static std::string temp_png_path(const std::string& base) { + std::stringstream path; + path << QDir::tempPath().toStdString() << "/libopenshot_" << base << "_" + << getpid() << "_" << rand() << ".png"; + return path.str(); +} + +static std::string create_source_png(int w, int h, const QColor& color) { + const std::string path = temp_png_path("source"); + QImage image(w, h, QImage::Format_RGBA8888_Premultiplied); + image.fill(color); + REQUIRE(image.save(QString::fromStdString(path))); + return path; +} + +static std::string create_mask_png(const std::vector& gray_values) { + const std::string path = temp_png_path("mask"); + QImage mask(static_cast(gray_values.size()), 1, QImage::Format_RGBA8888_Premultiplied); + for (size_t i = 0; i < gray_values.size(); ++i) { + const int gray = gray_values[i]; + mask.setPixelColor(static_cast(i), 0, QColor(gray, gray, gray, 255)); + } + REQUIRE(mask.save(QString::fromStdString(path))); + return path; +} + +static std::string create_uniform_mask_png(int width, int height, int gray_value) { + const std::string path = temp_png_path("mask_uniform"); + QImage mask(width, height, QImage::Format_RGBA8888_Premultiplied); + mask.fill(QColor(gray_value, gray_value, gray_value, 255)); + REQUIRE(mask.save(QString::fromStdString(path))); + return path; +} + +class TrackingMaskReader : public ReaderBase { +private: + bool is_open = false; + CacheMemory cache; + int width = 2; + int height = 1; + +public: + std::vector requests; + + TrackingMaskReader(int fps_num, int fps_den, int64_t length_frames) { + info.has_video = true; + info.has_audio = false; + info.has_single_image = false; + info.width = width; + info.height = height; + info.fps = Fraction(fps_num, fps_den); + info.video_length = length_frames; + info.duration = static_cast(length_frames / info.fps.ToDouble()); + info.sample_rate = 48000; + info.channels = 2; + info.audio_stream_index = -1; + } + + openshot::CacheBase* GetCache() override { return &cache; } + bool IsOpen() override { return is_open; } + std::string Name() override { return "TrackingMaskReader"; } + void Open() override { is_open = true; } + void Close() override { is_open = false; } + + std::shared_ptr GetFrame(int64_t number) override { + requests.push_back(number); + auto frame = std::make_shared(number, width, height, "#00000000"); + frame->GetImage()->fill(QColor(128, 128, 128, 255)); + return frame; + } + + std::string Json() const override { + return JsonValue().toStyledString(); + } + + Json::Value JsonValue() const override { + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "TrackingMaskReader"; + root["path"] = ""; + return root; + } + + void SetJson(const std::string value) override { + (void) value; + } + + void SetJsonValue(const Json::Value root) override { + ReaderBase::SetJsonValue(root); + } +}; + +static std::shared_ptr make_input_frame(int64_t number, int width = 2, int height = 1) { + auto frame = std::make_shared(number, width, height, "#00000000"); + frame->GetImage()->fill(QColor(64, 64, 64, 255)); + return frame; +} + +TEST_CASE("EffectBase common mask blend applies to ProcessFrame", "[effect][mask][base]") { + auto frame = std::make_shared(1, 4, 1, "#000000"); + auto image = frame->GetImage(); + image->fill(QColor(80, 80, 80, 255)); + + const std::string mask_path = create_mask_png({255, 255, 0, 0}); + + Brightness effect(Keyframe(0.5), Keyframe(0.0)); + effect.MaskReader(new QtImageReader(mask_path)); + + auto out = effect.ProcessFrame(frame, 1); + auto out_image = out->GetImage(); + + CHECK(out_image->pixelColor(0, 0).red() > 80); + CHECK(out_image->pixelColor(1, 0).red() > 80); + CHECK(out_image->pixelColor(2, 0).red() == 80); + CHECK(out_image->pixelColor(3, 0).red() == 80); +} + +TEST_CASE("EffectBase mask fields serialize and deserialize", "[effect][mask][json]") { + const std::string mask_path = create_mask_png({255, 0}); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.mask_invert = true; + effect.MaskReader(new QtImageReader(mask_path)); + + const Json::Value json = effect.JsonValue(); + CHECK(json["mask_invert"].asBool()); + REQUIRE(json["mask_reader"].isObject()); + CHECK(json["mask_reader"]["type"].asString() == "QtImageReader"); + + Brightness copy(Keyframe(0.0), Keyframe(0.0)); + copy.SetJsonValue(json); + CHECK(copy.mask_invert); + CHECK(copy.MaskReader() != nullptr); +} + +TEST_CASE("Blur mask mode drive amount differs from post blend", "[effect][mask][blur]") { + const int width = 20; + const int height = 20; + auto frame_post = std::make_shared(1, width, height, "#000000"); + auto frame_drive = std::make_shared(1, width, height, "#000000"); + + QImage input(width, height, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + const int alpha = std::min(255, x * 12 + y * 3); + input.setPixelColor(x, y, QColor(255, 180, 40, alpha)); + } + } + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_uniform_mask_png(width, height, 128); + + Blur post(Keyframe(3.0), Keyframe(3.0), Keyframe(3.0), Keyframe(1.0)); + post.mask_mode = BLUR_MASK_POST_BLEND; + post.MaskReader(new QtImageReader(mask_path)); + + Blur drive(Keyframe(3.0), Keyframe(3.0), Keyframe(3.0), Keyframe(1.0)); + drive.mask_mode = BLUR_MASK_DRIVE_AMOUNT; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + bool any_diff = false; + for (int y = 0; y < height && !any_diff; ++y) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, y) != out_drive->GetImage()->pixelColor(x, y)) { + any_diff = true; + break; + } + } + } + if (!any_diff) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, height / 2) != out_drive->GetImage()->pixelColor(x, height / 2)) { + any_diff = true; + break; + } + } + } + CHECK(any_diff); +} + +TEST_CASE("Saturation mask mode drive amount differs from post blend", "[effect][mask][saturation]") { + auto frame_post = std::make_shared(1, 1, 1, "#000000"); + auto frame_drive = std::make_shared(1, 1, 1, "#000000"); + + QImage input(1, 1, QImage::Format_RGBA8888_Premultiplied); + input.setPixelColor(0, 0, QColor(70, 120, 200, 255)); + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_mask_png({128}); + + Saturation post(Keyframe(2.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + post.mask_mode = SATURATION_MASK_POST_BLEND; + post.MaskReader(new QtImageReader(mask_path)); + + Saturation drive(Keyframe(2.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + drive.mask_mode = SATURATION_MASK_DRIVE_AMOUNT; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + CHECK(out_post->GetImage()->pixelColor(0, 0) != out_drive->GetImage()->pixelColor(0, 0)); +} + +TEST_CASE("Brightness mask mode vary strength differs from limit-to-area", "[effect][mask][brightness]") { + auto frame_post = std::make_shared(1, 1, 1, "#000000"); + auto frame_drive = std::make_shared(1, 1, 1, "#000000"); + + QImage input(1, 1, QImage::Format_RGBA8888_Premultiplied); + input.setPixelColor(0, 0, QColor(80, 120, 200, 255)); + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_mask_png({128}); + + Brightness post(Keyframe(0.6), Keyframe(6.0)); + post.mask_mode = BRIGHTNESS_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Brightness drive(Keyframe(0.6), Keyframe(6.0)); + drive.mask_mode = BRIGHTNESS_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + CHECK(out_post->GetImage()->pixelColor(0, 0) != out_drive->GetImage()->pixelColor(0, 0)); +} + +TEST_CASE("Hue mask mode vary strength differs from limit-to-area", "[effect][mask][hue]") { + auto frame_post = std::make_shared(1, 1, 1, "#000000"); + auto frame_drive = std::make_shared(1, 1, 1, "#000000"); + + QImage input(1, 1, QImage::Format_RGBA8888_Premultiplied); + input.setPixelColor(0, 0, QColor(200, 80, 40, 255)); + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_mask_png({128}); + + Hue post(Keyframe(0.8)); + post.mask_mode = HUE_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Hue drive(Keyframe(0.8)); + drive.mask_mode = HUE_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + CHECK(out_post->GetImage()->pixelColor(0, 0) != out_drive->GetImage()->pixelColor(0, 0)); +} + +TEST_CASE("Pixelate mask mode vary strength differs from limit-to-area", "[effect][mask][pixelate]") { + const int width = 20; + const int height = 20; + auto frame_post = std::make_shared(1, width, height, "#000000"); + auto frame_drive = std::make_shared(1, width, height, "#000000"); + + QImage input(width, height, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + input.setPixelColor(x, y, QColor((x * 13) % 256, (y * 11) % 256, ((x + y) * 7) % 256, 255)); + } + } + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_uniform_mask_png(width, height, 128); + + Pixelate post(Keyframe(1.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0)); + post.mask_mode = PIXELATE_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Pixelate drive(Keyframe(1.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0)); + drive.mask_mode = PIXELATE_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + bool any_diff = false; + for (int y = 0; y < height && !any_diff; ++y) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, y) != out_drive->GetImage()->pixelColor(x, y)) { + any_diff = true; + break; + } + } + } + CHECK(any_diff); +} + +TEST_CASE("Sharpen mask mode vary strength differs from limit-to-area", "[effect][mask][sharpen]") { + const int width = 20; + const int height = 20; + auto frame_post = std::make_shared(1, width, height, "#000000"); + auto frame_drive = std::make_shared(1, width, height, "#000000"); + + QImage input(width, height, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + const int red = (x < width / 2) ? 30 : 220; + const int green = (y < height / 2) ? 60 : 200; + const int blue = ((x + y) % 2 == 0) ? 20 : 240; + input.setPixelColor(x, y, QColor(red, green, blue, 255)); + } + } + *frame_post->GetImage() = input; + *frame_drive->GetImage() = input; + + const std::string mask_path = create_uniform_mask_png(width, height, 128); + + Sharpen post(Keyframe(1.6), Keyframe(3.0), Keyframe(0.05)); + post.mask_mode = SHARPEN_MASK_LIMIT_TO_AREA; + post.MaskReader(new QtImageReader(mask_path)); + + Sharpen drive(Keyframe(1.6), Keyframe(3.0), Keyframe(0.05)); + drive.mask_mode = SHARPEN_MASK_VARY_STRENGTH; + drive.MaskReader(new QtImageReader(mask_path)); + + auto out_post = post.ProcessFrame(frame_post, 1); + auto out_drive = drive.ProcessFrame(frame_drive, 1); + + bool any_diff = false; + for (int y = 0; y < height && !any_diff; ++y) { + for (int x = 0; x < width; ++x) { + if (out_post->GetImage()->pixelColor(x, y) != out_drive->GetImage()->pixelColor(x, y)) { + any_diff = true; + break; + } + } + } + CHECK(any_diff); +} + +TEST_CASE("Effect mask_mode roundtrip for supported effects", "[effect][mask][mode][json]") { + Blur blur(Keyframe(1.0), Keyframe(1.0), Keyframe(3.0), Keyframe(1.0)); + blur.mask_mode = BLUR_MASK_DRIVE_AMOUNT; + Blur blur_copy; + blur_copy.SetJsonValue(blur.JsonValue()); + CHECK(blur_copy.mask_mode == BLUR_MASK_DRIVE_AMOUNT); + + Saturation saturation(Keyframe(1.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + saturation.mask_mode = SATURATION_MASK_DRIVE_AMOUNT; + Saturation saturation_copy; + saturation_copy.SetJsonValue(saturation.JsonValue()); + CHECK(saturation_copy.mask_mode == SATURATION_MASK_DRIVE_AMOUNT); + + Brightness brightness(Keyframe(0.4), Keyframe(5.0)); + brightness.mask_mode = BRIGHTNESS_MASK_VARY_STRENGTH; + Brightness brightness_copy; + brightness_copy.SetJsonValue(brightness.JsonValue()); + CHECK(brightness_copy.mask_mode == BRIGHTNESS_MASK_VARY_STRENGTH); + + Hue hue(Keyframe(0.4)); + hue.mask_mode = HUE_MASK_VARY_STRENGTH; + Hue hue_copy; + hue_copy.SetJsonValue(hue.JsonValue()); + CHECK(hue_copy.mask_mode == HUE_MASK_VARY_STRENGTH); + + Pixelate pixelate(Keyframe(0.8), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0), Keyframe(0.0)); + pixelate.mask_mode = PIXELATE_MASK_VARY_STRENGTH; + Pixelate pixelate_copy; + pixelate_copy.SetJsonValue(pixelate.JsonValue()); + CHECK(pixelate_copy.mask_mode == PIXELATE_MASK_VARY_STRENGTH); + + Sharpen sharpen(Keyframe(1.5), Keyframe(2.0), Keyframe(0.1)); + sharpen.mask_mode = SHARPEN_MASK_VARY_STRENGTH; + Sharpen sharpen_copy; + sharpen_copy.SetJsonValue(sharpen.JsonValue()); + CHECK(sharpen_copy.mask_mode == SHARPEN_MASK_VARY_STRENGTH); +} + +TEST_CASE("EffectBase accepts legacy reader key for mask source", "[effect][mask][json][legacy_reader]") { + const std::string mask_path = create_mask_png({255, 0}); + QtImageReader reader(mask_path); + + Saturation effect(Keyframe(2.0), Keyframe(1.0), Keyframe(1.0), Keyframe(1.0)); + Json::Value update; + update["reader"] = reader.JsonValue(); + effect.SetJsonValue(update); + + REQUIRE(effect.MaskReader() != nullptr); + CHECK(effect.JsonValue()["mask_reader"]["type"].asString() == "QtImageReader"); +} + +TEST_CASE("EffectBase uses ClipBase start and end for mask source trim", "[effect][mask][timing][json]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + + Json::Value update; + update["start"] = -10; + update["end"] = -20; + update["mask_time_mode"] = 99; + update["mask_loop_mode"] = 99; + effect.SetJsonValue(update); + + const Json::Value clamped = effect.JsonValue(); + CHECK(clamped["start"].asDouble() == Approx(-10.0).margin(0.00001)); + CHECK(clamped["end"].asDouble() == Approx(-20.0).margin(0.00001)); + CHECK(clamped["mask_time_mode"].asInt() == 1); + CHECK(clamped["mask_loop_mode"].asInt() == 0); + + update["start"] = 0.5; + update["end"] = 1.2; + update["mask_time_mode"] = 1; + update["mask_loop_mode"] = 2; + effect.SetJsonValue(update); + const Json::Value roundtrip = effect.JsonValue(); + CHECK(roundtrip["start"].asDouble() == Approx(0.5).margin(0.00001)); + CHECK(roundtrip["end"].asDouble() == Approx(1.2).margin(0.00001)); + CHECK(roundtrip["mask_time_mode"].asInt() == 1); + CHECK(roundtrip["mask_loop_mode"].asInt() == 2); +} + +TEST_CASE("EffectBase defaults mask time mode to source FPS", "[effect][mask][timing][default]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + CHECK(effect.JsonValue()["mask_time_mode"].asInt() == 1); +} + +TEST_CASE("EffectBase mask properties use base start and end controls", "[effect][mask][timing][properties]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + Json::Value update; + update["start"] = 0.2; + update["end"] = 0.9; + update["mask_time_mode"] = 1; + update["mask_loop_mode"] = 2; + effect.SetJsonValue(update); + + const Json::Value properties = openshot::stringToJson(effect.PropertiesJSON(1)); + REQUIRE(properties.isObject()); + CHECK(properties["start"]["name"].asString() == "Start"); + CHECK(properties["end"]["name"].asString() == "End"); + CHECK(properties["mask_reader"]["name"].asString() == "Mask: Source"); + CHECK(properties["mask_time_mode"]["name"].asString() == "Mask: Time Mode"); + CHECK(properties["mask_loop_mode"]["name"].asString() == "Mask: Loop"); + CHECK_FALSE(properties.isMember("mask_start")); + CHECK_FALSE(properties.isMember("mask_end")); + CHECK(properties["mask_time_mode"]["choices"].size() == 2); + CHECK(properties["mask_loop_mode"]["choices"].size() == 3); +} + +TEST_CASE("EffectBase accepts legacy mask_start and mask_end aliases", "[effect][mask][timing][legacy_aliases]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + + Json::Value update; + update["mask_start"] = 0.5; + update["mask_end"] = 1.2; + effect.SetJsonValue(update); + + const Json::Value roundtrip = effect.JsonValue(); + CHECK(roundtrip["start"].asDouble() == Approx(0.5).margin(0.00001)); + CHECK(roundtrip["end"].asDouble() == Approx(1.2).margin(0.00001)); + CHECK_FALSE(roundtrip.isMember("mask_start")); + CHECK_FALSE(roundtrip.isMember("mask_end")); +} + +TEST_CASE("EffectBase canonical start and end override legacy mask aliases", "[effect][mask][timing][legacy_precedence]") { + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + + Json::Value update; + update["start"] = 0.25; + update["end"] = 0.75; + update["mask_start"] = 1.5; + update["mask_end"] = 2.5; + effect.SetJsonValue(update); + + const Json::Value roundtrip = effect.JsonValue(); + CHECK(roundtrip["start"].asDouble() == Approx(0.25).margin(0.00001)); + CHECK(roundtrip["end"].asDouble() == Approx(0.75).margin(0.00001)); +} + +TEST_CASE("EffectBase timeline mode maps one-to-one with repeat loop", "[effect][mask][timing][timeline][repeat]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + + Json::Value update; + update["start"] = 1.0 / 24.0; + update["end"] = 3.0 / 24.0; + update["mask_time_mode"] = 0; // Timeline + update["mask_loop_mode"] = 1; // Repeat + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 7; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 2, 3, 4, 2}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("EffectBase timeline mode supports ping-pong loop", "[effect][mask][timing][timeline][pingpong]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + + Json::Value update; + update["start"] = 1.0 / 24.0; + update["end"] = 3.0 / 24.0; + update["mask_time_mode"] = 0; // Timeline + update["mask_loop_mode"] = 2; // Ping-Pong + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 7; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 3, 2, 3, 4}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("EffectBase timeline mode play-once clamps at end", "[effect][mask][timing][timeline][once]") { + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.MaskReader(tracking); + + Json::Value update; + update["start"] = 1.0 / 24.0; + update["end"] = 3.0 / 24.0; + update["mask_time_mode"] = 0; // Timeline + update["mask_loop_mode"] = 0; // Play Once + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 6; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 4, 4, 4}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("EffectBase source FPS mode maps using parent clip FPS", "[effect][mask][timing][source_fps]") { + DummyReader clip_reader(Fraction(30, 1), 320, 240, 48000, 2, 4.0f); + Clip parent_clip; + parent_clip.Reader(&clip_reader); + + auto* tracking = new TrackingMaskReader(15, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.ParentClip(&parent_clip); + effect.MaskReader(tracking); + + Json::Value update; + update["start"] = 0.0; + update["end"] = 0; + update["mask_time_mode"] = 1; // Source FPS + update["mask_loop_mode"] = 0; // Play Once + effect.SetJsonValue(update); + + for (int64_t n = 1; n <= 5; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {1, 2, 2, 3, 3}; + CHECK(tracking->requests == expected); +} + +TEST_CASE("Clip-attached effects still process every frame while using trimmed mask source", "[effect][mask][timing][clip_effect]") { + DummyReader clip_reader(Fraction(24, 1), 320, 240, 48000, 2, 4.0f); + Clip parent_clip; + parent_clip.Reader(&clip_reader); + + auto* tracking = new TrackingMaskReader(24, 1, 100); + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.ParentClip(&parent_clip); + effect.MaskReader(tracking); + effect.Start(1.0 / 24.0); + effect.End(3.0 / 24.0); + effect.mask_loop_mode = EffectBase::MASK_LOOP_PLAY_ONCE; + + for (int64_t n = 1; n <= 6; ++n) + effect.ProcessFrame(make_input_frame(n), n); + + const std::vector expected = {2, 3, 4, 4, 4, 4}; + CHECK(tracking->requests == expected); +} diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index 1b1ff252..b31b0b12 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -13,6 +13,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include "openshot_catch.h" @@ -24,10 +30,74 @@ using namespace openshot; +namespace { + +double SampleAverageLuma(const std::shared_ptr& frame, int sample_grid = 4) { + const int width = frame->GetWidth(); + const int height = frame->GetHeight(); + if (width <= 0 || height <= 0) { + return 0.0; + } + + int64_t luma_sum = 0; + int64_t sample_count = 0; + for (int y = 0; y < sample_grid; ++y) { + const int row = std::min(height - 1, (y * height) / sample_grid); + const unsigned char* pixels = frame->GetPixels(row); + for (int x = 0; x < sample_grid; ++x) { + const int col = std::min(width - 1, (x * width) / sample_grid); + const int pixel_index = col * 4; + luma_sum += (pixels[pixel_index] + pixels[pixel_index + 1] + pixels[pixel_index + 2]) / 3; + ++sample_count; + } + } + + return sample_count > 0 + ? static_cast(luma_sum) / static_cast(sample_count) + : 0.0; +} + +struct HardwareDecoderSettingsGuard { + int decoder = 0; + int device = 0; + + HardwareDecoderSettingsGuard() + : decoder(Settings::Instance()->HARDWARE_DECODER), + device(Settings::Instance()->HW_DE_DEVICE_SET) {} + + ~HardwareDecoderSettingsGuard() { + Settings::Instance()->HARDWARE_DECODER = decoder; + Settings::Instance()->HW_DE_DEVICE_SET = device; + } +}; + +struct TemporaryFileGuard { + std::string path; + + explicit TemporaryFileGuard(std::string temp_path) + : path(std::move(temp_path)) {} + + ~TemporaryFileGuard() { + if (!path.empty()) { + std::remove(path.c_str()); + } + } +}; + +} + TEST_CASE( "Invalid_Path", "[libopenshot][ffmpegreader]" ) { - // Check invalid path - CHECK_THROWS_AS(FFmpegReader(""), InvalidFile); + // Check invalid path and error details + const std::string invalid_path = "/tmp/__openshot_missing_test_file__.mp4"; + try { + FFmpegReader r(invalid_path); + FAIL("Expected InvalidFile for missing media path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("FFmpegReader could not open media file.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } } TEST_CASE( "GetFrame_Before_Opening", "[libopenshot][ffmpegreader]" ) @@ -348,6 +418,244 @@ TEST_CASE( "Multiple_Open_and_Close", "[libopenshot][ffmpegreader]" ) r.Close(); } +TEST_CASE( "Static_Image_PNG_Reports_Single_Image", "[libopenshot][ffmpegreader]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "front.png"; + FFmpegReader r(path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + CHECK(r.info.has_video); + CHECK_FALSE(r.info.has_audio); + CHECK(r.info.has_single_image); + CHECK(r.info.video_length > 1); + CHECK(r.info.duration > 1000.0f); + + auto f1 = r.GetFrame(1); + auto f2 = r.GetFrame(std::min(2, static_cast(r.info.video_length))); + CHECK(f1->CheckPixel(50, 50, + f2->GetPixels(50)[50 * 4 + 0], + f2->GetPixels(50)[50 * 4 + 1], + f2->GetPixels(50)[50 * 4 + 2], + f2->GetPixels(50)[50 * 4 + 3], + 0)); + + r.Close(); +} + +TEST_CASE( "Static_Image_JPG_Reports_Single_Image", "[libopenshot][ffmpegreader]" ) +{ + // Generate a JPG fixture at runtime from a known PNG frame. + std::stringstream png_path; + png_path << TEST_MEDIA_PATH << "front.png"; + FFmpegReader png_reader(png_path.str()); + png_reader.Open(); + + auto png_frame = png_reader.GetFrame(1); + std::srand(static_cast(std::time(nullptr))); + std::stringstream jpg_path; + jpg_path << "libopenshot-static-image-test-" << std::rand() << ".jpg"; + REQUIRE(png_frame->GetImage()->save(jpg_path.str().c_str(), "JPG")); + png_reader.Close(); + + FFmpegReader jpg_reader(jpg_path.str(), DurationStrategy::VideoPreferred); + jpg_reader.Open(); + + CHECK(jpg_reader.info.has_video); + CHECK_FALSE(jpg_reader.info.has_audio); + CHECK(jpg_reader.info.has_single_image); + CHECK(jpg_reader.info.video_length > 1); + CHECK(jpg_reader.info.duration > 1000.0f); + + auto f1 = jpg_reader.GetFrame(1); + auto f2 = jpg_reader.GetFrame(std::min(2, static_cast(jpg_reader.info.video_length))); + CHECK(f1->CheckPixel(50, 50, + f2->GetPixels(50)[50 * 4 + 0], + f2->GetPixels(50)[50 * 4 + 1], + f2->GetPixels(50)[50 * 4 + 2], + f2->GetPixels(50)[50 * 4 + 3], + 2)); + + jpg_reader.Close(); + std::remove(jpg_path.str().c_str()); +} + +TEST_CASE( "Attached_Picture_Audio_Does_Not_Stall_Early_Frames", "[libopenshot][ffmpegreader]" ) +{ + // Build a temporary fixture with audio + attached cover art at runtime. + // This avoids adding another binary media file to the repository. + if (std::system("ffmpeg -hide_banner -version >/dev/null 2>&1") != 0) { + WARN("Skipping attached-picture test: ffmpeg executable not available"); + return; + } + + std::srand(static_cast(std::time(nullptr))); + std::stringstream fixture_path; + fixture_path << "libopenshot-attached-art-test-" << std::rand() << ".m4a"; + + std::stringstream command; + command << "ffmpeg -y -hide_banner -loglevel error " + << "-i \"" << TEST_MEDIA_PATH << "front.png\" " + << "-f lavfi -i \"anullsrc=r=44100:cl=stereo\" " + << "-t 2 " + << "-map 1:a:0 -map 0:v:0 " + << "-c:a aac -b:a 128k " + << "-c:v mjpeg -disposition:v:0 attached_pic " + << "\"" << fixture_path.str() << "\""; + const int command_result = std::system(command.str().c_str()); + REQUIRE(command_result == 0); + + FFmpegReader r(fixture_path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + CHECK(r.info.has_video); + CHECK(r.info.has_audio); + CHECK(r.info.has_single_image); + + auto f1 = r.GetFrame(1); + CHECK(f1->has_image_data); + CHECK(f1->GetAudioSamplesCount() > 0); + + const auto frame2_start = std::chrono::steady_clock::now(); + auto f2 = r.GetFrame(2); + const auto frame2_end = std::chrono::steady_clock::now(); + const auto frame2_ms = std::chrono::duration_cast(frame2_end - frame2_start).count(); + CHECK(frame2_ms < 1500); + CHECK(f2->has_image_data); + CHECK(f2->GetAudioSamplesCount() > 0); + + r.Close(); + std::remove(fixture_path.str().c_str()); +} + +TEST_CASE( "HardwareDecodeSuccessful_IsFalse_WhenHardwareDecodeIsDisabled", "[libopenshot][ffmpegreader][hardware]" ) +{ + HardwareDecoderSettingsGuard guard; + Settings::Instance()->HARDWARE_DECODER = 0; + Settings::Instance()->HW_DE_DEVICE_SET = 0; + + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader r(path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + REQUIRE(r.info.has_video); + auto frame = r.GetFrame(1); + REQUIRE(frame->has_image_data); + CHECK_FALSE(r.HardwareDecodeSuccessful()); + + r.Close(); +} + +TEST_CASE( "VAAPI_H264_420_Reports_HardwareDecodeSuccess", "[libopenshot][ffmpegreader][hardware]" ) +{ +#if !defined(__linux__) || !USE_HW_ACCEL + WARN("Skipping hardware decode success test: requires Linux build with hardware decode support"); + return; +#else + if (std::system("ffmpeg -hide_banner -version >/dev/null 2>&1") != 0) { + WARN("Skipping hardware decode success test: ffmpeg executable not available"); + return; + } + if (std::system("ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -q '\\'") != 0) { + WARN("Skipping hardware decode success test: ffmpeg does not report VAAPI support"); + return; + } + if (std::system("sh -c 'test -e /dev/dri/renderD128 -o -e /dev/dri/renderD129 -o -e /dev/dri/renderD130' >/dev/null 2>&1") != 0) { + WARN("Skipping hardware decode success test: no render node available under /dev/dri"); + return; + } + + std::srand(static_cast(std::time(nullptr))); + std::stringstream fixture_path; + fixture_path << "libopenshot-vaapi-420-test-" << std::rand() << ".mp4"; + TemporaryFileGuard fixture_cleanup(fixture_path.str()); + + std::stringstream command; + command << "ffmpeg -y -hide_banner -loglevel error " + << "-f lavfi -i \"testsrc2=size=128x72:rate=30\" " + << "-t 1 " + << "-c:v libx264 " + << "-pix_fmt yuv420p " + << "-profile:v high " + << "\"" << fixture_path.str() << "\""; + const int command_result = std::system(command.str().c_str()); + REQUIRE(command_result == 0); + + HardwareDecoderSettingsGuard hw_guard; + Settings::Instance()->HARDWARE_DECODER = 1; + Settings::Instance()->HW_DE_DEVICE_SET = 0; + + FFmpegReader r(fixture_path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + REQUIRE(r.info.has_video); + auto frame = r.GetFrame(1); + REQUIRE(frame->has_image_data); + CHECK(r.HardwareDecodeSuccessful()); + + r.Close(); +#endif +} + +TEST_CASE( "VAAPI_H264_422_Does_Not_Return_Black_Frames", "[libopenshot][ffmpegreader][hardware]" ) +{ +#if !defined(__linux__) || !USE_HW_ACCEL + WARN("Skipping VAAPI regression test: requires Linux build with hardware decode support"); + return; +#else + if (std::system("ffmpeg -hide_banner -version >/dev/null 2>&1") != 0) { + WARN("Skipping VAAPI regression test: ffmpeg executable not available"); + return; + } + if (std::system("ffmpeg -hide_banner -hwaccels 2>/dev/null | grep -q '\\'") != 0) { + WARN("Skipping VAAPI regression test: ffmpeg does not report VAAPI support"); + return; + } + if (std::system("sh -c 'test -e /dev/dri/renderD128 -o -e /dev/dri/renderD129 -o -e /dev/dri/renderD130' >/dev/null 2>&1") != 0) { + WARN("Skipping VAAPI regression test: no render node available under /dev/dri"); + return; + } + + std::srand(static_cast(std::time(nullptr))); + std::stringstream fixture_path; + fixture_path << "libopenshot-vaapi-422-test-" << std::rand() << ".mp4"; + TemporaryFileGuard fixture_cleanup(fixture_path.str()); + + std::stringstream command; + command << "ffmpeg -y -hide_banner -loglevel error " + << "-f lavfi -i \"testsrc2=size=128x72:rate=30\" " + << "-t 1 " + << "-c:v libx264 " + << "-pix_fmt yuvj422p " + << "-profile:v high422 " + << "-color_range pc " + << "\"" << fixture_path.str() << "\""; + const int command_result = std::system(command.str().c_str()); + REQUIRE(command_result == 0); + + HardwareDecoderSettingsGuard guard; + Settings::Instance()->HARDWARE_DECODER = 1; + Settings::Instance()->HW_DE_DEVICE_SET = 0; + + FFmpegReader r(fixture_path.str(), DurationStrategy::VideoPreferred); + r.Open(); + + REQUIRE(r.info.has_video); + REQUIRE(r.info.video_length >= 3); + + const std::array frames_to_check = {1, r.info.video_length / 2, r.info.video_length}; + for (const int64_t frame_number : frames_to_check) { + auto frame = r.GetFrame(frame_number); + REQUIRE(frame->has_image_data); + INFO("frame=" << frame_number << ", avg_luma=" << SampleAverageLuma(frame)); + CHECK(SampleAverageLuma(frame) > 8.0); + } + + r.Close(); +#endif +} + TEST_CASE( "verify parent Timeline", "[libopenshot][ffmpegreader]" ) { // Create a reader @@ -476,4 +784,4 @@ TEST_CASE( "Decoding AV1 Video", "[libopenshot][ffmpegreader]" ) } catch (const InvalidFile & e) { // Ignore older FFmpeg versions which don't support AV1 } -} \ No newline at end of file +} diff --git a/tests/FFmpegWriter.cpp b/tests/FFmpegWriter.cpp index 1eb89c79..8e8a66c6 100644 --- a/tests/FFmpegWriter.cpp +++ b/tests/FFmpegWriter.cpp @@ -12,6 +12,7 @@ #include #include +#include #include "openshot_catch.h" @@ -233,3 +234,75 @@ TEST_CASE( "Gif", "[libopenshot][ffmpegwriter]" ) // Close reader r1.Close(); } + +TEST_CASE( "SizeOrdering_x264_CRF", "[libopenshot][ffmpegwriter][filesize]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); + + auto file_size = [](const std::string& file_path) -> std::streamoff { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + if (!in) + return -1; + return in.tellg(); + }; + + auto encode = [&](const std::string& out_name, int crf, int audio_bitrate) -> std::streamoff { + FFmpegWriter w(out_name); + w.SetAudioOptions(true, "aac", 48000, 2, LAYOUT_STEREO, audio_bitrate); + w.SetVideoOptions(true, "libx264", Fraction(24,1), 1280, 720, Fraction(1,1), false, false, crf); + w.PrepareStreams(); + w.SetOption(VIDEO_STREAM, "crf", std::to_string(crf)); + w.Open(); + w.WriteFrame(&reader, 1, 120); + w.Close(); + return file_size(out_name); + }; + + const auto low_size = encode("SizeOrdering_x264_CRF_low.mp4", 30, 96000); + const auto med_size = encode("SizeOrdering_x264_CRF_med.mp4", 23, 128000); + const auto high_size = encode("SizeOrdering_x264_CRF_high.mp4", 20, 160000); + + CHECK(low_size < med_size); + CHECK(med_size < high_size); + + reader.Close(); +} + +TEST_CASE( "SizeOrdering_vp9_CRF", "[libopenshot][ffmpegwriter][filesize]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); + + auto file_size = [](const std::string& file_path) -> std::streamoff { + std::ifstream in(file_path, std::ios::binary | std::ios::ate); + if (!in) + return -1; + return in.tellg(); + }; + + auto encode = [&](const std::string& out_name, int crf, int audio_bitrate) -> std::streamoff { + FFmpegWriter w(out_name); + w.SetAudioOptions(true, "libvorbis", 48000, 2, LAYOUT_STEREO, audio_bitrate); + w.SetVideoOptions(true, "libvpx-vp9", Fraction(24,1), 1280, 720, Fraction(1,1), false, false, crf); + w.PrepareStreams(); + w.SetOption(VIDEO_STREAM, "crf", std::to_string(crf)); + w.Open(); + w.WriteFrame(&reader, 1, 120); + w.Close(); + return file_size(out_name); + }; + + const auto low_size = encode("SizeOrdering_vp9_CRF_low.webm", 46, 96000); + const auto med_size = encode("SizeOrdering_vp9_CRF_med.webm", 34, 128000); + const auto high_size = encode("SizeOrdering_vp9_CRF_high.webm", 26, 160000); + + CHECK(low_size < med_size); + CHECK(med_size < high_size); + + reader.Close(); +} diff --git a/tests/ImageReader.cpp b/tests/ImageReader.cpp index 9232fb08..8e81d08c 100644 --- a/tests/ImageReader.cpp +++ b/tests/ImageReader.cpp @@ -19,6 +19,20 @@ using namespace openshot; +TEST_CASE( "Invalid_Path_ImageReader", "[libopenshot][imagereader]" ) +{ + // Check invalid path and error details + const std::string invalid_path = "/tmp/__openshot_missing_test_file__.png"; + try { + ImageReader r(invalid_path); + FAIL("Expected InvalidFile for missing image path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("ImageReader could not open image file.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } +} + TEST_CASE( "Duration_And_Length_ImageReader", "[libopenshot][imagereader]" ) { // Create a reader diff --git a/tests/Mask.cpp b/tests/Mask.cpp new file mode 100644 index 00000000..9b22ca05 --- /dev/null +++ b/tests/Mask.cpp @@ -0,0 +1,206 @@ +/** + * @file + * @brief Unit tests for Mask effect behavior and reader compatibility + */ + +// Copyright (c) 2008-2026 OpenShot Studios, LLC +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "Frame.h" +#include "effects/Mask.h" +#include "QtImageReader.h" +#include "openshot_catch.h" + +using namespace openshot; + +static std::string temp_png_path(const std::string& base) { + std::stringstream path; + path << QDir::tempPath().toStdString() << "/libopenshot_" << base << "_" + << getpid() << "_" << rand() << ".png"; + return path.str(); +} + +static std::string create_mask_png(const std::vector& gray_values) { + const std::string path = temp_png_path("mask_effect"); + QImage mask(static_cast(gray_values.size()), 1, QImage::Format_RGBA8888_Premultiplied); + for (size_t i = 0; i < gray_values.size(); ++i) { + const int gray = gray_values[i]; + mask.setPixelColor(static_cast(i), 0, QColor(gray, gray, gray, 255)); + } + REQUIRE(mask.save(QString::fromStdString(path))); + return path; +} + +TEST_CASE("Mask applies alpha from reader source", "[effect][mask_effect]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(255, 0, 0, 255)); + image->setPixelColor(1, 0, QColor(255, 0, 0, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.GetFrame(frame, 1); + + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 0); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 255); +} + +TEST_CASE("Mask invert flips reader mask alpha mapping", "[effect][mask_effect][invert]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(255, 0, 0, 255)); + image->setPixelColor(1, 0, QColor(255, 0, 0, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.mask_invert = true; + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.GetFrame(frame, 1); + + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 255); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} + +TEST_CASE("Mask replace_image emits grayscale values", "[effect][mask_effect][replace]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + frame->GetImage()->fill(QColor(10, 20, 30, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.replace_image = true; + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.GetFrame(frame, 1); + auto px0 = out->GetImage()->pixelColor(0, 0); + auto px1 = out->GetImage()->pixelColor(1, 0); + + CHECK(px0.red() == px0.green()); + CHECK(px0.green() == px0.blue()); + CHECK(px1.red() == px1.green()); + CHECK(px1.green() == px1.blue()); + CHECK(px0.alpha() == px0.red()); + CHECK(px1.alpha() == px1.red()); +} + +TEST_CASE("Mask accepts legacy reader json field", "[effect][mask_effect][json]") { + const std::string mask_path = create_mask_png({128}); + QtImageReader reader(mask_path); + + Json::Value root; + root["reader"] = reader.JsonValue(); + root["brightness"] = Keyframe(0.0).JsonValue(); + root["contrast"] = Keyframe(0.0).JsonValue(); + + Mask mask; + mask.SetJsonValue(root); + + REQUIRE(mask.Reader() != nullptr); + CHECK(mask.JsonValue().isMember("mask_reader")); + CHECK(mask.JsonValue()["mask_reader"]["type"].asString() == "QtImageReader"); +} + +TEST_CASE("Mask legacy start and end json load into base trim", "[effect][mask_effect][json][timing]") { + Json::Value root; + root["start"] = 0.5; + root["end"] = 1.25; + root["brightness"] = Keyframe(0.0).JsonValue(); + root["contrast"] = Keyframe(0.0).JsonValue(); + + Mask mask; + mask.SetJsonValue(root); + + CHECK(mask.Start() == Approx(0.5).margin(0.00001)); + CHECK(mask.End() == Approx(1.25).margin(0.00001)); + CHECK(mask.JsonValue()["start"].asDouble() == Approx(0.5).margin(0.00001)); + CHECK(mask.JsonValue()["end"].asDouble() == Approx(1.25).margin(0.00001)); +} + +TEST_CASE("Mask ProcessFrame brightness 1.0 fully clears output", "[effect][mask_effect][process][brightness]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(255, 10, 10, 255)); + image->setPixelColor(1, 0, QColor(255, 10, 10, 255)); + + const std::string mask_path = create_mask_png({255, 255}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(1.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 0); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} + +TEST_CASE("Mask ProcessFrame honors invert mask property", "[effect][mask_effect][process][invert]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(80, 40, 20, 255)); + image->setPixelColor(1, 0, QColor(80, 40, 20, 255)); + + const std::string mask_path = create_mask_png({255, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.mask_invert = true; + mask.brightness = Keyframe(0.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 255); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} + +TEST_CASE("Mask ProcessFrame brightness -1.0 keeps output opaque", "[effect][mask_effect][process][brightness]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(20, 200, 20, 255)); + image->setPixelColor(1, 0, QColor(20, 200, 20, 255)); + + const std::string mask_path = create_mask_png({0, 0}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(-1.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 255); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 255); +} + +TEST_CASE("Mask ProcessFrame brightness 1.0 ignores gray mask and still clears", "[effect][mask_effect][process][brightness]") { + auto frame = std::make_shared(1, 2, 1, "#000000"); + auto image = frame->GetImage(); + image->setPixelColor(0, 0, QColor(180, 80, 30, 255)); + image->setPixelColor(1, 0, QColor(180, 80, 30, 255)); + + const std::string mask_path = create_mask_png({128, 128}); + Mask mask; + mask.Reader(new QtImageReader(mask_path)); + mask.brightness = Keyframe(1.0); + mask.contrast = Keyframe(0.0); + + auto out = mask.ProcessFrame(frame, 1); + CHECK(out->GetImage()->pixelColor(0, 0).alpha() == 0); + CHECK(out->GetImage()->pixelColor(1, 0).alpha() == 0); +} diff --git a/tests/Profiles.cpp b/tests/Profiles.cpp index 8e319a3b..6c6e827d 100644 --- a/tests/Profiles.cpp +++ b/tests/Profiles.cpp @@ -11,11 +11,40 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include "openshot_catch.h" +#include #include +#include +#include +#include "Exceptions.h" #include "Profiles.h" +static std::string get_temp_test_path(const std::string& file_name) { +#ifdef _WIN32 + const char* base = std::getenv("TEMP"); + if (!base || !*base) { + base = std::getenv("TMP"); + } + if (!base || !*base) { + base = "."; + } + return std::string(base) + "\\" + file_name; +#else + const char* base = std::getenv("TMPDIR"); + if (!base || !*base) { + base = "/tmp"; + } + return std::string(base) + "/" + file_name; +#endif +} + +static std::string test_output_profile_path(const std::string& base_name) { + std::stringstream file_name; + file_name << base_name << "_" << std::rand() << ".profile"; + return get_temp_test_path(file_name.str()); +} + TEST_CASE( "empty constructor", "[libopenshot][profile]" ) { openshot::Profile p1; @@ -90,6 +119,38 @@ TEST_CASE( "constructor with example profiles", "[libopenshot][profile]" ) CHECK(p2.info.spherical == false); } +TEST_CASE( "invalid profile path message", "[libopenshot][profile]" ) +{ + const std::string invalid_path = get_temp_test_path("__openshot_missing_test_profile__"); + std::remove(invalid_path.c_str()); + try { + openshot::Profile p(invalid_path); + FAIL("Expected InvalidFile for missing profile path"); + } catch (const openshot::InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("Profile file could not be found or opened.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } +} + +TEST_CASE( "invalid profile parse message", "[libopenshot][profile]" ) +{ + const std::string invalid_profile = get_temp_test_path("openshot_invalid_profile_for_test.profile"); + { + std::ofstream f(invalid_profile); + f << "width=abc\n"; + } + + try { + openshot::Profile p(invalid_profile); + FAIL("Expected InvalidFile for malformed profile contents"); + } catch (const openshot::InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("Profile file could not be parsed (invalid format or values).") != std::string::npos); + CHECK(message.find(invalid_profile) != std::string::npos); + } +} + TEST_CASE( "24 fps names", "[libopenshot][profile]" ) { std::stringstream path; @@ -166,12 +227,11 @@ TEST_CASE( "save profiles", "[libopenshot][profile]" ) openshot::Profile p1(profile1.str()); // Save copy - std::stringstream profile1_copy; - profile1_copy << TEST_MEDIA_PATH << "example_profile1_copy"; - p1.Save(profile1_copy.str()); + const std::string profile1_copy = test_output_profile_path("example_profile1_copy"); + p1.Save(profile1_copy); // Load saved copy - openshot::Profile p1_load_copy(profile1_copy.str()); + openshot::Profile p1_load_copy(profile1_copy); // Default values CHECK(p1_load_copy.info.description == "HD 720p 24 fps"); @@ -218,12 +278,11 @@ TEST_CASE( "spherical profiles", "[libopenshot][profile]" ) CHECK(p_json.ShortName() == "3840x1920p30 360°"); // Save and reload to test file I/O - std::stringstream profile_path; - profile_path << TEST_MEDIA_PATH << "example_profile_360"; - p.Save(profile_path.str()); + const std::string profile_path = test_output_profile_path("example_profile_360"); + p.Save(profile_path); // Load the saved profile - openshot::Profile p_loaded(profile_path.str()); + openshot::Profile p_loaded(profile_path); CHECK(p_loaded.info.spherical == true); CHECK(p_loaded.ShortName() == "3840x1920p30 360°"); @@ -232,4 +291,4 @@ TEST_CASE( "spherical profiles", "[libopenshot][profile]" ) p_non_spherical.info.spherical = false; CHECK_FALSE(p == p_non_spherical); -} \ No newline at end of file +} diff --git a/tests/QtImageReader.cpp b/tests/QtImageReader.cpp index e48839c3..6c39774b 100644 --- a/tests/QtImageReader.cpp +++ b/tests/QtImageReader.cpp @@ -25,8 +25,16 @@ using namespace openshot; TEST_CASE( "Default_Constructor", "[libopenshot][qtimagereader]" ) { - // Check invalid path - CHECK_THROWS_AS(QtImageReader(""), InvalidFile); + // Check invalid path and error details + const std::string invalid_path = "/tmp/__openshot_missing_test_file__.png"; + try { + QtImageReader r(invalid_path); + FAIL("Expected InvalidFile for missing image path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("QtImageReader could not open image file.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } } TEST_CASE( "GetFrame_Before_Opening", "[libopenshot][qtimagereader]" ) diff --git a/tests/Timeline.cpp b/tests/Timeline.cpp index 0c90fbd7..3d9a0c94 100644 --- a/tests/Timeline.cpp +++ b/tests/Timeline.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include "openshot_catch.h" @@ -21,13 +23,133 @@ #include "FrameMapper.h" #include "Timeline.h" #include "Clip.h" +#include "CacheMemory.h" +#include "DummyReader.h" #include "Frame.h" #include "Fraction.h" +#include "effects/Brightness.h" +#include "Exceptions.h" #include "effects/Blur.h" +#include "effects/Bars.h" #include "effects/Negate.h" using namespace openshot; +static uint64_t image_fingerprint(const std::shared_ptr& image) { + const uint64_t kFnvOffset = 1469598103934665603ULL; + const uint64_t kFnvPrime = 1099511628211ULL; + uint64_t hash = kFnvOffset; + + if (!image) { + return hash; + } + + const unsigned char* bytes = image->constBits(); + const size_t count = static_cast(image->sizeInBytes()); + for (size_t i = 0; i < count; ++i) { + hash ^= static_cast(bytes[i]); + hash *= kFnvPrime; + } + + return hash; +} + +class TimelineTrackingMaskReader : public ReaderBase { +private: + bool is_open = false; + CacheMemory cache; + int width = 2; + int height = 1; + +public: + std::vector requests; + + TimelineTrackingMaskReader(int fps_num, int fps_den, int64_t length_frames) { + info.has_video = true; + info.has_audio = false; + info.width = width; + info.height = height; + info.fps = Fraction(fps_num, fps_den); + info.video_length = length_frames; + info.duration = static_cast(length_frames / info.fps.ToDouble()); + info.sample_rate = 48000; + info.channels = 2; + info.audio_stream_index = -1; + } + + openshot::CacheBase* GetCache() override { return &cache; } + bool IsOpen() override { return is_open; } + std::string Name() override { return "TimelineTrackingMaskReader"; } + void Open() override { is_open = true; } + void Close() override { is_open = false; } + + std::shared_ptr GetFrame(int64_t number) override { + requests.push_back(number); + auto frame = std::make_shared(number, width, height, "#00000000"); + frame->GetImage()->fill(QColor(128, 128, 128, 255)); + return frame; + } + + std::string Json() const override { return JsonValue().toStyledString(); } + Json::Value JsonValue() const override { + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "TimelineTrackingMaskReader"; + root["path"] = ""; + return root; + } + void SetJson(const std::string value) override { (void) value; } + void SetJsonValue(const Json::Value root) override { ReaderBase::SetJsonValue(root); } +}; + +class TimelineSolidColorReader : public ReaderBase { +private: + bool is_open = false; + CacheMemory cache; + QColor color; + +public: + TimelineSolidColorReader(int width, + int height, + int fps_num, + int fps_den, + int64_t length_frames, + const QColor& fill_color) + : color(fill_color) { + info.has_video = true; + info.has_audio = false; + info.width = width; + info.height = height; + info.fps = Fraction(fps_num, fps_den); + info.video_length = length_frames; + info.duration = static_cast(length_frames / info.fps.ToDouble()); + info.sample_rate = 48000; + info.channels = 2; + info.audio_stream_index = -1; + } + + openshot::CacheBase* GetCache() override { return &cache; } + bool IsOpen() override { return is_open; } + std::string Name() override { return "TimelineSolidColorReader"; } + void Open() override { is_open = true; } + void Close() override { is_open = false; } + + std::shared_ptr GetFrame(int64_t number) override { + auto frame = std::make_shared(number, info.width, info.height, "#00000000"); + frame->GetImage()->fill(color); + return frame; + } + + std::string Json() const override { return JsonValue().toStyledString(); } + Json::Value JsonValue() const override { + Json::Value root = ReaderBase::JsonValue(); + root["type"] = "TimelineSolidColorReader"; + root["path"] = ""; + return root; + } + void SetJson(const std::string value) override { (void) value; } + void SetJsonValue(const Json::Value root) override { ReaderBase::SetJsonValue(root); } +}; + TEST_CASE( "constructor", "[libopenshot][timeline]" ) { Fraction fps(30000,1000); @@ -44,6 +166,19 @@ TEST_CASE( "constructor", "[libopenshot][timeline]" ) CHECK(t2.info.height == 240); } +TEST_CASE( "project constructor invalid path message", "[libopenshot][timeline]" ) +{ + const std::string invalid_path = "/tmp/__openshot_missing_test_project__.osp"; + try { + Timeline t(invalid_path, true); + FAIL("Expected InvalidFile for missing timeline project path"); + } catch (const InvalidFile& e) { + const std::string message = e.what(); + CHECK(message.find("Timeline project file could not be opened.") != std::string::npos); + CHECK(message.find(invalid_path) != std::string::npos); + } +} + TEST_CASE( "Set Json and clear clips", "[libopenshot][timeline]" ) { Fraction fps(30000,1000); @@ -600,6 +735,88 @@ TEST_CASE( "Effect: Blur", "[libopenshot][timeline]" ) t.Close(); } +TEST_CASE("Global mask effect source FPS mode follows timeline FPS mapping", "[libopenshot][timeline][effect][mask][timing]") { + Timeline t(320, 240, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + + DummyReader clip_reader(Fraction(30, 1), 320, 240, 44100, 2, 2.0f); + Clip clip(&clip_reader); + clip.Layer(0); + clip.Position(0.0); + clip.Start(0.0); + clip.End(1.0); + t.AddClip(&clip); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.Layer(0); + effect.Position(0.0); + effect.Start(0.0); + effect.End(1.0); + + auto* tracking = new TimelineTrackingMaskReader(15, 1, 120); + effect.MaskReader(tracking); + + Json::Value timing; + timing["mask_time_mode"] = 1; // Source FPS + timing["mask_loop_mode"] = 0; // Play Once + timing["start"] = 0.0; + timing["end"] = 1.0; + effect.SetJsonValue(timing); + + t.AddEffect(&effect); + t.Open(); + + for (int64_t frame = 1; frame <= 5; ++frame) { + auto out = t.GetFrame(frame); + REQUIRE(out != nullptr); + } + + const std::vector expected = {1, 2, 2, 3, 3}; + CHECK(tracking->requests == expected); + + t.Close(); +} + +TEST_CASE("Global mask effect start trims source without freezing playback", "[libopenshot][timeline][effect][mask][trim]") { + Timeline t(320, 240, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + + DummyReader clip_reader(Fraction(30, 1), 320, 240, 44100, 2, 2.0f); + Clip clip(&clip_reader); + clip.Layer(0); + clip.Position(0.0); + clip.Start(0.0); + clip.End(1.0); + t.AddClip(&clip); + + Brightness effect(Keyframe(0.0), Keyframe(0.0)); + effect.Layer(0); + effect.Position(0.0); + effect.Start(1.0 / 15.0); + effect.End(1.0); + + auto* tracking = new TimelineTrackingMaskReader(15, 1, 120); + effect.MaskReader(tracking); + + Json::Value timing; + timing["mask_time_mode"] = 1; // Source FPS + timing["mask_loop_mode"] = 0; // Play Once + timing["start"] = 1.0 / 15.0; + timing["end"] = 1.0; + effect.SetJsonValue(timing); + + t.AddEffect(&effect); + t.Open(); + + for (int64_t frame = 1; frame <= 5; ++frame) { + auto out = t.GetFrame(frame); + REQUIRE(out != nullptr); + } + + const std::vector expected = {2, 3, 3, 4, 4}; + CHECK(tracking->requests == expected); + + t.Close(); +} + TEST_CASE( "GetMaxFrame and GetMaxTime", "[libopenshot][timeline]" ) { // Create a timeline @@ -1067,6 +1284,223 @@ TEST_CASE( "ApplyJSONDiff insert invalidates overlapping timeline cache", "[libo CHECK(!t.GetCache()->Contains(10)); } +TEST_CASE( "ApplyJSONDiff alpha updates refresh fixed-frame preview content", "[libopenshot][timeline]" ) +{ + // Deterministic solid-color readers avoid any fixture/image ambiguity. + TimelineSolidColorReader base_reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(10, 200, 20, 255) + ); + TimelineSolidColorReader overlay_reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(220, 30, 180, 255) + ); + + Clip base_clip(&base_reader); + base_clip.Id("BASE_ALPHA_TEST"); + base_clip.Layer(0); + base_clip.Position(0.0); + base_clip.End(5.0); + + Clip overlay_clip(&overlay_reader); + overlay_clip.Id("OVERLAY_ALPHA_TEST"); + overlay_clip.Layer(1); + overlay_clip.Position(0.0); + overlay_clip.End(5.0); + + Timeline t(64, 64, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&base_clip); + t.AddClip(&overlay_clip); + t.Open(); + + const int64_t frame_number = 1; + + auto apply_alpha = [&](double alpha_value) { + Json::Value root(Json::arrayValue); + Json::Value change(Json::objectValue); + change["type"] = "update"; + change["partial"] = true; + + Json::Value key(Json::arrayValue); + key.append("clips"); + Json::Value key_id(Json::objectValue); + key_id["id"] = overlay_clip.Id(); + key.append(key_id); + change["key"] = key; + + Json::Value alpha_json(Json::objectValue); + Json::Value points(Json::arrayValue); + Json::Value p1(Json::objectValue); + p1["co"]["X"] = 1.0; + p1["co"]["Y"] = 1.0; + p1["interpolation"] = 0; + points.append(p1); + Json::Value p2(Json::objectValue); + p2["co"]["X"] = static_cast(frame_number); + p2["co"]["Y"] = alpha_value; + p2["interpolation"] = 1; + points.append(p2); + alpha_json["Points"] = points; + + Json::Value value(Json::objectValue); + value["alpha"] = alpha_json; + change["value"] = value; + + root.append(change); + t.ApplyJsonDiff(root.toStyledString()); + + Clip* updated = t.GetClip(overlay_clip.Id()); + REQUIRE(updated != nullptr); + CHECK(updated->alpha.GetValue(frame_number) == Approx(alpha_value).margin(0.0001)); + }; + + // Establish reference colors for alpha=1.0 (top) and alpha=0.0 (bottom). + // Prime cache at fixed frame. + std::shared_ptr initial = t.GetFrame(frame_number); + REQUIRE(initial != nullptr); + REQUIRE(t.GetCache() != nullptr); + REQUIRE(overlay_clip.GetCache() != nullptr); + REQUIRE(t.GetCache()->Contains(frame_number)); + REQUIRE(overlay_clip.GetCache()->Count() > 0); + + // Repeated alpha updates at the same frame must invalidate both timeline and + // clip caches, preventing stale preview frames from being reused. + const std::vector alpha_steps = {0.9, 0.8, 0.7, 0.6, 0.5}; + for (double alpha_value : alpha_steps) { + apply_alpha(alpha_value); + CHECK(!t.GetCache()->Contains(frame_number)); + CHECK(overlay_clip.GetCache()->Count() == 0); + + // Re-request frame to repopulate caches before next update. + std::shared_ptr refreshed = t.GetFrame(frame_number); + REQUIRE(refreshed != nullptr); + CHECK(t.GetCache()->Contains(frame_number)); + CHECK(overlay_clip.GetCache()->Count() > 0); + } +} + +TEST_CASE( "ApplyJSONDiff enables and drains edit safety window", "[libopenshot][timeline][edit]" ) +{ + TimelineSolidColorReader reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(220, 30, 180, 255) + ); + + Clip clip(&reader); + clip.Id("EDIT_WINDOW_CLIP"); + clip.Layer(0); + clip.Position(0.0); + clip.End(10.0); + + Timeline t(64, 64, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&clip); + t.Open(); + + CHECK(t.SafeEditFramesRemaining() == 0); + + Json::Value root(Json::arrayValue); + Json::Value change(Json::objectValue); + change["type"] = "update"; + change["partial"] = true; + + Json::Value key(Json::arrayValue); + key.append("clips"); + Json::Value key_id(Json::objectValue); + key_id["id"] = clip.Id(); + key.append(key_id); + change["key"] = key; + + Json::Value value(Json::objectValue); + value["rotation"] = Keyframe(10.0).JsonValue(); + change["value"] = value; + root.append(change); + + t.ApplyJsonDiff(root.toStyledString()); + CHECK(t.SafeEditFramesRemaining() == 240); + + (void)t.GetFrame(1); + CHECK(t.SafeEditFramesRemaining() == 239); + + for (int64_t frame = 2; frame <= 240; ++frame) { + (void)t.GetFrame(frame); + } + CHECK(t.SafeEditFramesRemaining() == 0); +} + +TEST_CASE( "ApplyJSONDiff clip Bars effect updates refresh fixed-frame preview content", "[libopenshot][timeline][effect][bars]" ) +{ + TimelineSolidColorReader base_reader( + /*width=*/64, /*height=*/64, /*fps_num=*/30, /*fps_den=*/1, /*length_frames=*/300, + QColor(10, 200, 20, 255) + ); + + Clip clip(&base_reader); + clip.Id("BARS_CLIP_TEST"); + clip.Layer(0); + clip.Position(0.0); + clip.End(5.0); + + Bars bars; + bars.Id("BARS_EFFECT_TEST"); + bars.Layer(0); + bars.Position(0.0); + bars.Start(0.0); + bars.End(5.0); + bars.color = Color("#000000"); + bars.left = Keyframe(0.0); + bars.top = Keyframe(0.0); + bars.right = Keyframe(0.0); + bars.bottom = Keyframe(0.0); + clip.AddEffect(&bars); + + Timeline t(64, 64, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&clip); + t.Open(); + + const int64_t frame_number = 1; + auto frame = t.GetFrame(frame_number); + REQUIRE(frame != nullptr); + CHECK(frame->GetImage()->pixelColor(20, 20) == QColor(10, 200, 20, 255)); + uint64_t previous_hash = image_fingerprint(frame->GetImage()); + + const std::vector top_steps = {0.02, 0.04, 0.06, 0.08, 0.10}; + for (double top_value : top_steps) { + Keyframe top_kf(top_value); + + Json::Value root(Json::arrayValue); + Json::Value change(Json::objectValue); + change["type"] = "update"; + change["partial"] = true; + + Json::Value key(Json::arrayValue); + key.append("clips"); + Json::Value clip_key(Json::objectValue); + clip_key["id"] = clip.Id(); + key.append(clip_key); + key.append("effects"); + Json::Value effect_key(Json::objectValue); + effect_key["id"] = bars.Id(); + key.append(effect_key); + change["key"] = key; + + Json::Value value(Json::objectValue); + value["top"] = top_kf.JsonValue(); + change["value"] = value; + root.append(change); + + t.ApplyJsonDiff(root.toStyledString()); + CHECK(bars.top.GetValue(frame_number) == Approx(top_value).margin(0.0001)); + + frame = t.GetFrame(frame_number); + REQUIRE(frame != nullptr); + const uint64_t current_hash = image_fingerprint(frame->GetImage()); + + // Regression check: every Bars update should change the rendered image. + CHECK(current_hash != previous_hash); + previous_hash = current_hash; + } +} + TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) { // Create a timeline @@ -1140,3 +1574,34 @@ TEST_CASE( "ApplyJSONDiff Update Reader Info", "[libopenshot][timeline]" ) CHECK(mapper->Reader()->info.duration == Approx(20.88333).margin(0.00001)); } + +TEST_CASE("GetFrame past-end requests are not cached", "[libopenshot][timeline][cache]") { + TimelineSolidColorReader reader( + 64, 64, + 30, 1, + 300, + QColor(10, 20, 30, 255)); + Clip clip(&reader); + clip.Layer(1); + clip.Position(0.0); + clip.End(1.0); + + Timeline t(640, 480, Fraction(30, 1), 44100, 2, LAYOUT_STEREO); + t.AddClip(&clip); + t.Open(); + + const int64_t end = t.GetMaxFrame(); + REQUIRE(end > 1); + REQUIRE(t.GetCache() != nullptr); + const int64_t count_before = t.GetCache()->Count(); + + std::shared_ptr first = t.GetFrame(end + 25); + REQUIRE(first != nullptr); + CHECK(first->number == end + 25); + CHECK(t.GetCache()->Count() == count_before); + + std::shared_ptr second = t.GetFrame(end + 120); + REQUIRE(second != nullptr); + CHECK(second->number == end + 120); + CHECK(t.GetCache()->Count() == count_before); +} diff --git a/tests/VideoCacheThread.cpp b/tests/VideoCacheThread.cpp index 5493a141..60cab97d 100644 --- a/tests/VideoCacheThread.cpp +++ b/tests/VideoCacheThread.cpp @@ -42,6 +42,11 @@ public: void setMinFramesAhead(int64_t v) { min_frames_ahead.store(v); } void setLastDir(int d) { last_dir.store(d); } void forceUserSeekFlag() { userSeeked.store(true); } + bool isScrubbing() const { return scrub_active.load(); } + bool getUserSeekedFlag() const { return userSeeked.load(); } + bool getPrerollOnNextFill() const { return preroll_on_next_fill.load(); } + bool getClearCacheOnNextFill() const { return clear_cache_on_next_fill.load(); } + int64_t getRequestedDisplayFrame() const { return requested_display_frame.load(); } }; // ---------------------------------------------------------------------------- @@ -159,6 +164,26 @@ TEST_CASE("isReady: clamps preroll requirement at timeline boundaries", "[VideoC CHECK(thread.isReady()); } +TEST_CASE("isReady: treats out-of-range playhead as timeline edge for readiness", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(30,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 10); + + thread.setMinFramesAhead(30); + thread.setSpeed(1); + thread.setPlayhead(end + 100); + + thread.setLastCachedIndex(end - 1); + CHECK(!thread.isReady()); + thread.setLastCachedIndex(end); + CHECK(thread.isReady()); +} + TEST_CASE("clearCacheIfPaused: clears only when paused and not in cache", "[VideoCacheThread]") { TestableVideoCacheThread thread; CacheMemory cache(/*max_bytes=*/100000000); @@ -192,6 +217,48 @@ TEST_CASE("clearCacheIfPaused: clears only when paused and not in cache", "[Vide CHECK(cache.Contains(5)); } +TEST_CASE("clearCacheIfPaused: clears when paused past timeline end and playhead frame is missing", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(end - 1, 0, 0)); + const int64_t initial_count = cache.Count(); + REQUIRE(initial_count > 0); + + const bool didClear = thread.clearCacheIfPaused(/*playhead=*/end + 12, /*paused=*/true, &cache); + CHECK(didClear); + CHECK(cache.Count() == 0); +} + +TEST_CASE("clearCacheIfPaused: does not clear when paused past timeline end and end frame is cached", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(end, 0, 0)); + const int64_t initial_count = cache.Count(); + REQUIRE(initial_count > 0); + + const bool didClear = thread.clearCacheIfPaused(/*playhead=*/end + 12, /*paused=*/true, &cache); + CHECK(!didClear); + CHECK(cache.Count() == initial_count); +} + TEST_CASE("handleUserSeek: sets last_cached_index to playhead - dir", "[VideoCacheThread]") { TestableVideoCacheThread thread; @@ -313,3 +380,259 @@ TEST_CASE("prefetchWindow: interrupt on userSeeked flag", "[VideoCacheThread]") CHECK(thread.getLastCachedIndex() == 23); CHECK(!wasFull); } + +TEST_CASE("Seek preview: preserves playhead frame when paused and inside cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(100, 0, 0)); + cache.Add(std::make_shared(101, 0, 0)); + REQUIRE(cache.Count() >= 2); + + thread.Seek(/*new_position=*/100, /*start_preroll=*/false); + + CHECK(thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(cache.Contains(100)); + CHECK(cache.Count() >= 2); +} + +TEST_CASE("Seek preview: outside cache marks uncached without preroll", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(10, 0, 0)); + cache.Add(std::make_shared(11, 0, 0)); + REQUIRE(cache.Count() >= 2); + + thread.Seek(/*new_position=*/300, /*start_preroll=*/false); + + CHECK(thread.isScrubbing()); + CHECK(thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(cache.Count() >= 2); +} + +TEST_CASE("Seek commit: exits scrub mode and enables preroll when uncached", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + thread.Seek(/*new_position=*/200, /*start_preroll=*/false); + CHECK(thread.isScrubbing()); + + thread.Seek(/*new_position=*/200, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(thread.getUserSeekedFlag()); + CHECK(thread.getPrerollOnNextFill()); +} + +TEST_CASE("Seek commit: paused in-range seek preserves cached window state", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(120, 0, 0)); + cache.Add(std::make_shared(121, 0, 0)); + REQUIRE(cache.Count() >= 2); + + // Simulate existing cache progress so we can verify no baseline reset. + thread.setLastCachedIndex(180); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(thread.getLastCachedIndex() == 180); + CHECK(cache.Contains(120)); +} + +TEST_CASE("Seek commit: paused scrub preview then same-frame commit preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(140, 0, 0)); + cache.Add(std::make_shared(141, 0, 0)); + REQUIRE(cache.Count() >= 2); + + thread.setLastCachedIndex(210); + + // Typical paused seek flow: preview move, then commit same frame. + thread.Seek(/*new_position=*/140, /*start_preroll=*/false); + REQUIRE(thread.isScrubbing()); + REQUIRE(thread.getRequestedDisplayFrame() == 140); + + thread.Seek(/*new_position=*/140, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(thread.getLastCachedIndex() == 210); + CHECK(cache.Contains(140)); + CHECK(cache.Count() >= 2); +} + +TEST_CASE("Seek preview: paused out-of-range seek clamps to end and preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(end, 0, 0)); + cache.Add(std::make_shared(end - 1, 0, 0)); + thread.setLastCachedIndex(end - 1); + + thread.Seek(/*new_position=*/end + 24, /*start_preroll=*/false); + + CHECK(thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(thread.getRequestedDisplayFrame() == end + 24); + CHECK(thread.getLastCachedIndex() == end - 1); + CHECK(cache.Contains(end)); +} + +TEST_CASE("Seek commit: paused out-of-range seek past end enables cache rebuild when end is uncached", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + const int64_t end = timeline.info.video_length; + REQUIRE(end > 1); + + cache.Add(std::make_shared(1, 0, 0)); + cache.Add(std::make_shared(2, 0, 0)); + thread.setLastCachedIndex(2); + + thread.Seek(/*new_position=*/end + 24, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(thread.getUserSeekedFlag()); + CHECK(thread.getPrerollOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == end + 24); + CHECK(thread.getLastCachedIndex() == end - 1); + CHECK(!cache.Contains(end)); +} + +TEST_CASE("Seek commit: playback jump to cached frame preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(150, 0, 0)); + cache.Add(std::make_shared(220, 0, 0)); + thread.setPlayhead(220); + thread.setSpeed(1); + thread.setLastCachedIndex(230); + + thread.Seek(/*new_position=*/150, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(!thread.getClearCacheOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == 150); + CHECK(thread.getLastCachedIndex() == 230); +} + +TEST_CASE("Seek commit: playback click inside active cached window preserves cache", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(220, 0, 0)); + cache.Add(std::make_shared(230, 0, 0)); + cache.Add(std::make_shared(260, 0, 0)); + thread.setPlayhead(220); + thread.setSpeed(1); + thread.setLastCachedIndex(260); + + thread.Seek(/*new_position=*/230, /*start_preroll=*/true); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(!thread.getClearCacheOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == 230); + CHECK(thread.getLastCachedIndex() == 260); +} + +TEST_CASE("NotifyPlaybackPosition: ignored while scrubbing, applied after commit", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/false); + REQUIRE(thread.isScrubbing()); + CHECK(thread.getRequestedDisplayFrame() == 120); + + thread.NotifyPlaybackPosition(/*new_position=*/25); + CHECK(thread.getRequestedDisplayFrame() == 120); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/true); + REQUIRE(!thread.isScrubbing()); + + thread.NotifyPlaybackPosition(/*new_position=*/25); + CHECK(thread.getRequestedDisplayFrame() == 25); +} + +TEST_CASE("Seek non-preroll: playback uncached target does not force cache rebuild", "[VideoCacheThread]") { + TestableVideoCacheThread thread; + CacheMemory cache(/*max_bytes=*/100000000); + Timeline timeline(/*width=*/1280, /*height=*/720, /*fps=*/Fraction(24,1), + /*sample_rate=*/48000, /*channels=*/2, ChannelLayout::LAYOUT_STEREO); + timeline.SetCache(&cache); + thread.Reader(&timeline); + + cache.Add(std::make_shared(220, 0, 0)); + cache.Add(std::make_shared(221, 0, 0)); + thread.setPlayhead(220); + thread.setSpeed(1); + thread.setLastCachedIndex(230); + + thread.Seek(/*new_position=*/120, /*start_preroll=*/false); + + CHECK(!thread.isScrubbing()); + CHECK(!thread.getUserSeekedFlag()); + CHECK(!thread.getPrerollOnNextFill()); + CHECK(!thread.getClearCacheOnNextFill()); + CHECK(thread.getRequestedDisplayFrame() == 120); + CHECK(thread.getLastCachedIndex() == 230); +}