From 967b58c0d3f2876f18e4da3b44e03340bf035042 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 8 Mar 2026 00:31:21 -0600 Subject: [PATCH] - Added BASIC_SOFT and made it the default for new ChromaKey effects. - BASIC = hard cutoff; BASIC_SOFT = hard cutoff + halo feather. - Optimized loops/math + OpenMP, and added benchmark + tests for both and JSON/default behavior. --- src/Enums.h | 3 +- src/effects/ChromaKey.cpp | 203 ++++++++++++++++++++++++++++++-------- src/effects/ChromaKey.h | 7 +- tests/Benchmark.cpp | 35 +++++++ tests/ChromaKey.cpp | 63 ++++++++++++ 5 files changed, 265 insertions(+), 46 deletions(-) diff --git a/src/Enums.h b/src/Enums.h index cea6a1a2..8f3cb324 100644 --- a/src/Enums.h +++ b/src/Enums.h @@ -168,7 +168,8 @@ enum ChromaKeyMethod CHROMAKEY_CIE_LCH_H, ///< Difference between CIE LCH(ab) hues CHROMAKEY_CIE_DISTANCE, ///< CIEDE2000 perceptual difference CHROMAKEY_YCBCR, ///< YCbCr vector difference of CbCr - CHROMAKEY_LAST_METHOD = CHROMAKEY_YCBCR + CHROMAKEY_BASIC_SOFT, ///< BASIC metric + optional halo feathering + CHROMAKEY_LAST_METHOD = CHROMAKEY_BASIC_SOFT }; } // namespace openshot diff --git a/src/effects/ChromaKey.cpp b/src/effects/ChromaKey.cpp index 41421eac..0e54cd3a 100644 --- a/src/effects/ChromaKey.cpp +++ b/src/effects/ChromaKey.cpp @@ -16,13 +16,15 @@ #if USE_BABL #include #endif +#include +#include #include #include using namespace openshot; /// Blank constructor, useful when using Json to load the effect properties -ChromaKey::ChromaKey() : fuzz(5.0), halo(0), method(CHROMAKEY_BASIC) { +ChromaKey::ChromaKey() : fuzz(20.0), halo(10.0), method(CHROMAKEY_BASIC_SOFT) { // Init default color color = Color(); @@ -114,7 +116,8 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptr CHROMAKEY_BASIC && method <= CHROMAKEY_LAST_METHOD) + if (method != CHROMAKEY_BASIC && method != CHROMAKEY_BASIC_SOFT + && method <= CHROMAKEY_LAST_METHOD) { static bool need_init = true; @@ -282,33 +285,56 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptrscanLine(y); - - for (int x = 0; x < width; ++x, pixel += 4, pc += 3) + case CHROMAKEY_YCBCR: { - int db = (int) pc[1] - mask.u[1]; - int dr = (int) pc[2] - mask.u[2]; - float tmp = sqrt(db * db + dr * dr); + // sqrt(db^2 + dr^2), with db/dr in [-255, 255] + static const std::array sqrt_lut = [] { + std::array lut{}; + for (int i = 0; i <= 130050; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); - if (tmp <= threshold) - { - pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; - } - else if (tmp <= threshold + halothreshold) - { - float alphamult = (tmp - threshold) / halothreshold; + const int threshold_sq = threshold * threshold; + const int halo_upper = threshold + halothreshold; + const int halo_upper_sq = halo_upper * halo_upper; + const float inv_halo = (halothreshold > 0) ? (1.0f / halothreshold) : 0.0f; + const unsigned char key_cb = mask.u[1]; + const unsigned char key_cr = mask.u[2]; + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); - pixel[0] *= alphamult; - pixel[1] *= alphamult; - pixel[2] *= alphamult; - pixel[3] *= alphamult; + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char *pixel = img_bits + (y * img_bpl); + const unsigned char *pc_row = pixelbuf.data() + (y * rowwidth); + + for (int x = 0; x < width; ++x, pixel += 4) + { + const unsigned char *pc_px = pc_row + (x * 3); + int db = static_cast(pc_px[1]) - key_cb; + int dr = static_cast(pc_px[2]) - key_cr; + int dist_sq = db * db + dr * dr; + + if (dist_sq <= threshold_sq) + { + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + else if (halothreshold > 0 && dist_sq <= halo_upper_sq) + { + float tmp = sqrt_lut[dist_sq]; + float alphamult = (tmp - threshold) * inv_halo; + + pixel[0] *= alphamult; + pixel[1] *= alphamult; + pixel[2] *= alphamult; + pixel[3] *= alphamult; + } + } } } - } - break; + break; case CHROMAKEY_CIE_LCH_L: for (int y = 0; y < height; ++y) @@ -485,29 +511,121 @@ std::shared_ptr ChromaKey::GetFrame(std::shared_ptrscanLine(y); + // Metric upper bound for Color::GetDistance internal term is below 589825. + static const std::array dist_lut = [] { + std::array lut{}; + for (int i = 0; i <= 589825; ++i) + lut[i] = static_cast(std::sqrt(static_cast(i))); + return lut; + }(); + static const std::array inv_alpha = [] { + std::array lut{}; + lut[0] = 0.0f; + for (int i = 1; i < 256; ++i) + lut[i] = 255.0f / static_cast(i); + return lut; + }(); - for (int x = 0; x < width; ++x, pixel += 4) - { - float A = pixel[3]; - unsigned char R = (pixel[0] / A) * 255.0; - unsigned char G = (pixel[1] / A) * 255.0; - unsigned char B = (pixel[2] / A) * 255.0; + if (method == CHROMAKEY_BASIC) { + // Legacy BASIC behavior (hard threshold, no halo), optimized with OpenMP. + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); - // Get distance between mask color and pixel color - long distance = Color::GetDistance((long)R, (long)G, (long)B, mask_R, mask_G, mask_B); + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char * pixel = img_bits + (y * img_bpl); + for (int x = 0; x < width; ++x, pixel += 4) + { + const int A = pixel[3]; + if (A <= 0) + continue; - if (distance <= threshold) { - // MATCHED - Make pixel transparent - // Due to premultiplied alpha, we must also zero out - // the individual color channels (or else artifacts are left behind) - pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + // Preserve legacy 8-bit conversion behavior by casting to unsigned char. + unsigned char R = 0; + unsigned char G = 0; + unsigned char B = 0; + if (A == 255) { + R = pixel[0]; + G = pixel[1]; + B = pixel[2]; + } else { + const float inv_a = inv_alpha[A]; + R = static_cast(pixel[0] * inv_a); + G = static_cast(pixel[1] * inv_a); + B = static_cast(pixel[2] * inv_a); + } + + const int rmean = (static_cast(R) + static_cast(mask_R)) / 2; + const int r = static_cast(R) - static_cast(mask_R); + const int g = static_cast(G) - static_cast(mask_G); + const int b = static_cast(B) - static_cast(mask_B); + const int dist_sq = (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8); + const int distance = dist_lut[dist_sq]; + + if (distance <= threshold) + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + } + } else { + // BASIC_SOFT: same metric, optional halo feathering, optimized + OpenMP. + static const std::array dist_f_lut = [] { + std::array lut{}; + for (int i = 0; i <= 589825; ++i) + lut[i] = std::sqrt(static_cast(i)); + return lut; + }(); + const int halo_upper = threshold + halothreshold; + const int halo_upper_sq = halo_upper * halo_upper; + const float inv_halo = (halothreshold > 0) ? (1.0f / halothreshold) : 0.0f; + unsigned char *img_bits = image->bits(); + const int img_bpl = image->bytesPerLine(); + + #pragma omp parallel for schedule(static) + for (int y = 0; y < height; ++y) + { + unsigned char * pixel = img_bits + (y * img_bpl); + for (int x = 0; x < width; ++x, pixel += 4) + { + const int A = pixel[3]; + if (A <= 0) + continue; + + int R = 0; + int G = 0; + int B = 0; + if (A == 255) { + R = pixel[0]; + G = pixel[1]; + B = pixel[2]; + } else { + const float inv_a = inv_alpha[A]; + R = std::clamp(static_cast(pixel[0] * inv_a), 0, 255); + G = std::clamp(static_cast(pixel[1] * inv_a), 0, 255); + B = std::clamp(static_cast(pixel[2] * inv_a), 0, 255); + } + + const int rmean = (R + static_cast(mask_R)) / 2; + const int r = R - static_cast(mask_R); + const int g = G - static_cast(mask_G); + const int b = B - static_cast(mask_B); + const int dist_sq = (((512 + rmean) * r * r) >> 8) + 4 * g * g + (((767 - rmean) * b * b) >> 8); + + const int distance = dist_lut[dist_sq]; + if (distance <= threshold) { + pixel[0] = pixel[1] = pixel[2] = pixel[3] = 0; + } + else if (halothreshold > 0 && dist_sq <= halo_upper_sq) { + const float distance_f = dist_f_lut[dist_sq]; + const float alphamult = (distance_f - threshold) * inv_halo; + pixel[0] = static_cast(pixel[0] * alphamult); + pixel[1] = static_cast(pixel[1] * alphamult); + pixel[2] = static_cast(pixel[2] * alphamult); + pixel[3] = static_cast(pixel[3] * alphamult); + } + } } } - } // return the modified frame return frame; @@ -584,6 +702,7 @@ std::string ChromaKey::PropertiesJSON(int64_t requested_frame) const { root["halo"] = add_property_json("Halo", halo.GetValue(requested_frame), "float", "", &halo, 0, 125, false, requested_frame); root["keymethod"] = add_property_json("Key Method", method, "int", "", NULL, 0, CHROMAKEY_LAST_METHOD, false, requested_frame); root["keymethod"]["choices"].append(add_property_choice_json("Basic keying", 0, method)); + root["keymethod"]["choices"].append(add_property_choice_json("Basic keying (soft)", CHROMAKEY_BASIC_SOFT, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSV/HSL hue", 1, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSV saturation", 2, method)); root["keymethod"]["choices"].append(add_property_choice_json("HSL saturation", 3, method)); diff --git a/src/effects/ChromaKey.h b/src/effects/ChromaKey.h index 2a642988..b5dca0a0 100644 --- a/src/effects/ChromaKey.h +++ b/src/effects/ChromaKey.h @@ -56,13 +56,14 @@ namespace openshot /// /// The keying method determines the algorithm to use to determine the distance /// between the key color and the pixel color. The default keying method, - /// CHROMAKEY_BASIC, treates each of the R,G,B values as a vector and calculates + /// CHROMAKEY_BASIC_SOFT, uses the BASIC RGB-distance metric with optional halo feathering. + /// CHROMAKEY_BASIC treats each of the R,G,B values as a vector and calculates /// the length of the difference between those vectors. /// /// Pixels that are less than "fuzz" distance from the key color are eliminated /// by setting their alpha values to zero. /// - /// If halo is non-zero, pixels that are withing the halo distance of the fuzz + /// If halo is non-zero, pixels that are within the halo distance of the fuzz /// distance are given an alpha value that increases with the distance from the /// fuzz boundary. /// @@ -76,7 +77,7 @@ namespace openshot /// @param fuzz The fuzz factor (or threshold) /// @param halo The additional threshold for halo elimination. /// @param method The keying method - ChromaKey(Color color, Keyframe fuzz, Keyframe halo = 0.0, ChromaKeyMethod method = CHROMAKEY_BASIC); + ChromaKey(Color color, Keyframe fuzz, Keyframe halo = 0.0, ChromaKeyMethod method = CHROMAKEY_BASIC_SOFT); /// @brief This method is required for all derived classes of ClipBase, and returns a /// new openshot::Frame object. All Clip keyframes and effects are resolved into diff --git a/tests/Benchmark.cpp b/tests/Benchmark.cpp index f7e19c17..768fb4fa 100644 --- a/tests/Benchmark.cpp +++ b/tests/Benchmark.cpp @@ -28,6 +28,7 @@ #include "ReaderBase.h" #include "Timeline.h" #include "effects/Brightness.h" +#include "effects/ChromaKey.h" #include "effects/Crop.h" #include "effects/Mask.h" #include "effects/Saturation.h" @@ -63,6 +64,7 @@ int main(int argc, char* argv[]) { const string overlay = base + "front3.png"; string filter_test; bool list_only = false; + const int64_t chroma_bench_frames = 500; for (int i = 1; i < argc; ++i) { const string arg = argv[i]; @@ -243,6 +245,39 @@ int main(int argc, char* argv[]) { r.Close(); }); + trials.emplace_back("Effect_ChromaKey_BASIC", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + // Default/basic chroma key method baseline + ChromaKey key(Color(0, 255, 0, 255), Keyframe(80.0), Keyframe(20.0), CHROMAKEY_BASIC); + clip.AddEffect(&key); + const int64_t bench_frames = std::min(clip.info.video_length, chroma_bench_frames); + for (int64_t i = 1; i <= bench_frames; ++i) + clip.GetFrame(i); + for (int64_t i = bench_frames; i >= 1; --i) + clip.GetFrame(i); + clip.Close(); + r.Close(); + }); + + trials.emplace_back("Effect_ChromaKey_BASIC_SOFT", [&]() { + FFmpegReader r(video); + r.Open(); + Clip clip(&r); + clip.Open(); + ChromaKey key(Color(0, 255, 0, 255), Keyframe(80.0), Keyframe(20.0), CHROMAKEY_BASIC_SOFT); + clip.AddEffect(&key); + const int64_t bench_frames = std::min(clip.info.video_length, chroma_bench_frames); + for (int64_t i = 1; i <= bench_frames; ++i) + clip.GetFrame(i); + for (int64_t i = bench_frames; i >= 1; --i) + clip.GetFrame(i); + clip.Close(); + r.Close(); + }); + if (list_only) { for (const auto& trial : trials) cout << trial.first << "\n"; diff --git a/tests/ChromaKey.cpp b/tests/ChromaKey.cpp index c478bcba..af6e0827 100644 --- a/tests/ChromaKey.cpp +++ b/tests/ChromaKey.cpp @@ -69,3 +69,66 @@ TEST_CASE( "threshold", "[libopenshot][effect][chromakey]" ) CHECK(pix_e == expected); } +TEST_CASE( "default method is basic soft", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::ChromaKey e; + Json::Value json = e.JsonValue(); + CHECK(json["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} + +TEST_CASE( "basic vs basic soft halo behavior", "[libopenshot][effect][chromakey]" ) +{ + // Pick a green value in the halo band for key=(0,255,0), threshold=5, halo=20. + // For BASIC this should remain unchanged, for BASIC_SOFT it should be partially faded. + auto frame_basic = std::make_shared(1, 320, 180, "#00fa00"); + auto frame_soft = std::make_shared(1, 320, 180, "#00fa00"); + + openshot::Color key(0, 255, 0, 255); + openshot::Keyframe fuzz(5); + openshot::Keyframe halo(20); + + openshot::ChromaKey basic(key, fuzz, halo, CHROMAKEY_BASIC); + openshot::ChromaKey soft(key, fuzz, halo, CHROMAKEY_BASIC_SOFT); + + auto out_basic = basic.GetFrame(frame_basic, 1); + auto out_soft = soft.GetFrame(frame_soft, 1); + + QColor basic_pix = out_basic->GetImage()->pixelColor(10, 10); + QColor soft_pix = out_soft->GetImage()->pixelColor(10, 10); + + CHECK(basic_pix == QColor(0, 250, 0, 255)); + CHECK(soft_pix.alpha() < 255); + CHECK(soft_pix.alpha() > 0); + CHECK(soft_pix != basic_pix); +} + +TEST_CASE( "json roundtrip preserves method", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::Color key(0, 255, 0, 255); + openshot::Keyframe fuzz(5); + openshot::Keyframe halo(20); + openshot::ChromaKey basic(key, fuzz, halo, CHROMAKEY_BASIC); + openshot::ChromaKey soft(key, fuzz, halo, CHROMAKEY_BASIC_SOFT); + + Json::Value basic_json = basic.JsonValue(); + Json::Value soft_json = soft.JsonValue(); + + openshot::ChromaKey basic_loaded; + openshot::ChromaKey soft_loaded; + basic_loaded.SetJsonValue(basic_json); + soft_loaded.SetJsonValue(soft_json); + + CHECK(basic_loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC); + CHECK(soft_loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +} + +TEST_CASE( "SetJson string preserves keymethod", "[libopenshot][effect][chromakey][json]" ) +{ + openshot::ChromaKey source(openshot::Color(0, 255, 0, 255), openshot::Keyframe(5), openshot::Keyframe(20), CHROMAKEY_BASIC_SOFT); + const std::string payload = source.Json(); + + openshot::ChromaKey loaded; + loaded.SetJson(payload); + + CHECK(loaded.JsonValue()["keymethod"].asInt() == CHROMAKEY_BASIC_SOFT); +}