diff --git a/src/FrameScope.cpp b/src/FrameScope.cpp index dff08abb..f0c0cd4f 100644 --- a/src/FrameScope.cpp +++ b/src/FrameScope.cpp @@ -15,11 +15,14 @@ #include #include #include +#include using namespace openshot; namespace { constexpr float kInv255 = 1.0f / 255.0f; +constexpr float kVectorscopeUMax = 0.43600f; +constexpr float kVectorscopeVMax = 0.61500f; static int clamp_int(int value, int min_value, int max_value) { return std::max(min_value, std::min(max_value, value)); @@ -47,6 +50,13 @@ static Json::Value json_array_from_vector(const std::vector& values) { return array; } +static Json::Value json_array_from_vector(const std::vector& values) { + Json::Value array(Json::arrayValue); + for (size_t i = 0; i < values.size(); ++i) + array.append(Json::Value::UInt(values[i])); + return array; +} + static Json::Value json_array_from_vector(const std::vector& values) { Json::Value array(Json::arrayValue); for (size_t i = 0; i < values.size(); ++i) @@ -56,20 +66,46 @@ static Json::Value json_array_from_vector(const std::vector& values) { } FrameScope::FrameScope() - : frame(nullptr), waveform_columns(256), audio_buckets(256), waveform_bins(256), json_dirty(true) { + : frame(nullptr), + waveform_columns(256), + audio_buckets(256), + vectorscope_size(256), + roi_enabled(false), + roi_x(0.0f), + roi_y(0.0f), + roi_width(1.0f), + roi_height(1.0f), + waveform_bins(256), + waveform_column_map_width(0), + waveform_column_map_columns(0), + json_dirty(true) { reset(); } -FrameScope::FrameScope(std::shared_ptr new_frame, int new_waveform_columns, int new_audio_buckets) +FrameScope::FrameScope(std::shared_ptr new_frame, int new_waveform_columns, int new_audio_buckets, int new_vectorscope_size) : frame(new_frame), waveform_columns(std::max(1, new_waveform_columns)), audio_buckets(std::max(1, new_audio_buckets)), + vectorscope_size(std::max(1, new_vectorscope_size)), + roi_enabled(false), + roi_x(0.0f), + roi_y(0.0f), + roi_width(1.0f), + roi_height(1.0f), waveform_bins(256), + waveform_column_map_width(0), + waveform_column_map_columns(0), json_dirty(true) { analyze(); } void FrameScope::reset() { + reset_video(); + reset_audio(); + json_dirty = true; +} + +void FrameScope::reset_video() { video_present = false; video_width = 0; video_height = 0; @@ -79,15 +115,20 @@ void FrameScope::reset() { clipped_red = 0; clipped_green = 0; clipped_blue = 0; - histogram_luma.assign(256, 0); - histogram_red.assign(256, 0); - histogram_green.assign(256, 0); - histogram_blue.assign(256, 0); - waveform_luma.assign(static_cast(waveform_columns) * waveform_bins, 0); - waveform_red.assign(static_cast(waveform_columns) * waveform_bins, 0); - waveform_green.assign(static_cast(waveform_columns) * waveform_bins, 0); - waveform_blue.assign(static_cast(waveform_columns) * waveform_bins, 0); + ensure_video_buffers(); + std::fill(histogram_luma.begin(), histogram_luma.end(), 0u); + std::fill(histogram_red.begin(), histogram_red.end(), 0u); + std::fill(histogram_green.begin(), histogram_green.end(), 0u); + std::fill(histogram_blue.begin(), histogram_blue.end(), 0u); + std::fill(waveform_luma.begin(), waveform_luma.end(), 0u); + std::fill(waveform_red.begin(), waveform_red.end(), 0u); + std::fill(waveform_green.begin(), waveform_green.end(), 0u); + std::fill(waveform_blue.begin(), waveform_blue.end(), 0u); + std::fill(vectorscope.begin(), vectorscope.end(), 0u); + json_dirty = true; +} +void FrameScope::reset_audio() { audio_present = false; audio_channels = 0; audio_samples = 0; @@ -100,6 +141,40 @@ void FrameScope::reset() { json_dirty = true; } +void FrameScope::ensure_video_buffers() { + histogram_luma.resize(256); + histogram_red.resize(256); + histogram_green.resize(256); + histogram_blue.resize(256); + waveform_luma.resize(static_cast(waveform_columns) * static_cast(waveform_bins)); + waveform_red.resize(static_cast(waveform_columns) * static_cast(waveform_bins)); + waveform_green.resize(static_cast(waveform_columns) * static_cast(waveform_bins)); + waveform_blue.resize(static_cast(waveform_columns) * static_cast(waveform_bins)); + vectorscope.resize(static_cast(vectorscope_size) * static_cast(vectorscope_size)); +} + +void FrameScope::ensure_audio_buffers() { + audio_peak.assign(static_cast(audio_channels), 0.0f); + audio_rms.assign(static_cast(audio_channels), 0.0f); + audio_clipped_samples.assign(static_cast(audio_channels), 0u); + audio_waveform_min.assign(static_cast(audio_channels), std::vector(static_cast(audio_buckets), 0.0f)); + audio_waveform_max.assign(static_cast(audio_channels), std::vector(static_cast(audio_buckets), 0.0f)); +} + +void FrameScope::rebuild_waveform_column_map(int width) { + if (width == waveform_column_map_width && waveform_columns == waveform_column_map_columns && + static_cast(waveform_column_map.size()) == width) + return; + + waveform_column_map.resize(static_cast(width)); + const int waveform_column_limit = waveform_columns - 1; + for (int x = 0; x < width; ++x) + waveform_column_map[static_cast(x)] = clamp_int((x * waveform_columns) / std::max(1, width), 0, waveform_column_limit); + + waveform_column_map_width = width; + waveform_column_map_columns = waveform_columns; +} + void FrameScope::SetFrame(std::shared_ptr new_frame) { frame = new_frame; analyze(); @@ -107,12 +182,45 @@ void FrameScope::SetFrame(std::shared_ptr new_frame) { void FrameScope::SetWaveformColumns(int columns) { waveform_columns = std::max(1, columns); - analyze(); + reset_video(); + if (frame) + analyze_video(); } void FrameScope::SetAudioBuckets(int buckets) { audio_buckets = std::max(1, buckets); - analyze(); + reset_audio(); + if (frame) + analyze_audio(); +} + +void FrameScope::SetVectorscopeSize(int size) { + vectorscope_size = std::max(1, size); + reset_video(); + if (frame) + analyze_video(); +} + +void FrameScope::SetVideoRegionNormalized(float x, float y, float width, float height) { + roi_x = std::max(0.0f, std::min(1.0f, x)); + roi_y = std::max(0.0f, std::min(1.0f, y)); + roi_width = std::max(0.0f, std::min(1.0f - roi_x, width)); + roi_height = std::max(0.0f, std::min(1.0f - roi_y, height)); + roi_enabled = roi_width > 0.0f && roi_height > 0.0f; + reset_video(); + if (frame) + analyze_video(); +} + +void FrameScope::ClearVideoRegion() { + roi_enabled = false; + roi_x = 0.0f; + roi_y = 0.0f; + roi_width = 1.0f; + roi_height = 1.0f; + reset_video(); + if (frame) + analyze_video(); } void FrameScope::analyze() { @@ -133,29 +241,40 @@ void FrameScope::analyze_video() { return; video_present = true; - video_width = image->width(); - video_height = image->height(); + const int width = image->width(); + const int height = image->height(); + int start_x = 0; + int end_x = width; + int start_y = 0; + int end_y = height; + if (roi_enabled) { + start_x = clamp_int(static_cast(std::floor(roi_x * width)), 0, width - 1); + start_y = clamp_int(static_cast(std::floor(roi_y * height)), 0, height - 1); + end_x = clamp_int(static_cast(std::ceil((roi_x + roi_width) * width)), start_x + 1, width); + end_y = clamp_int(static_cast(std::ceil((roi_y + roi_height) * height)), start_y + 1, height); + } + video_width = std::max(1, end_x - start_x); + video_height = std::max(1, end_y - start_y); + ensure_video_buffers(); double luma_sum = 0.0; + double pixel_total = 0.0; clipped_shadows = 0; clipped_highlights = 0; clipped_red = 0; clipped_green = 0; clipped_blue = 0; - const int width = image->width(); - const int height = image->height(); const int bytes_per_line = image->bytesPerLine(); const unsigned char* bits = image->constBits(); const auto& inv_alpha = inv_alpha_lut(); - std::vector waveform_column_map(static_cast(width), 0); - const int waveform_column_limit = waveform_columns - 1; - for (int x = 0; x < width; ++x) - waveform_column_map[static_cast(x)] = clamp_int((x * waveform_columns) / std::max(1, width), 0, waveform_column_limit); + rebuild_waveform_column_map(video_width); + const float vectorscope_center = static_cast(vectorscope_size - 1) * 0.5f; + const float vectorscope_scale = vectorscope_center; - for (int y = 0; y < height; ++y) { + for (int y = start_y; y < end_y; ++y) { const unsigned char* row = bits + (static_cast(y) * bytes_per_line); - for (int x = 0; x < width; ++x) { + for (int x = start_x; x < end_x; ++x) { const unsigned char* pixel = row + (static_cast(x) * 4); const int red = pixel[0]; // RGBA8888: [R=0, G=1, B=2, A=3] const int green = pixel[1]; @@ -183,7 +302,15 @@ void FrameScope::analyze_video() { const int red_idx = byte_bin(redf); const int green_idx = byte_bin(greenf); const int blue_idx = byte_bin(bluef); - const size_t waveform_offset = static_cast(waveform_column_map[static_cast(x)]) * static_cast(waveform_bins); + const int roi_column = x - start_x; + const size_t waveform_offset = static_cast(waveform_column_map[static_cast(roi_column)]) * static_cast(waveform_bins); + const float u = -0.14713f * redf - 0.28886f * greenf + 0.43600f * bluef; + const float v = 0.61500f * redf - 0.51499f * greenf - 0.10001f * bluef; + const float normalized_u = u / kVectorscopeUMax; + const float normalized_v = v / kVectorscopeVMax; + const int vector_x = clamp_int(static_cast(std::round(vectorscope_center + (normalized_u * vectorscope_scale))), 0, vectorscope_size - 1); + const int vector_y = clamp_int(static_cast(std::round(vectorscope_center - (normalized_v * vectorscope_scale))), 0, vectorscope_size - 1); + const size_t vector_offset = (static_cast(vector_y) * static_cast(vectorscope_size)) + static_cast(vector_x); histogram_luma[luma_idx]++; histogram_red[red_idx]++; @@ -193,8 +320,10 @@ void FrameScope::analyze_video() { waveform_red[waveform_offset + red_idx]++; waveform_green[waveform_offset + green_idx]++; waveform_blue[waveform_offset + blue_idx]++; + vectorscope[vector_offset]++; luma_sum += luma; + pixel_total += 1.0; if (luma_idx <= 2) clipped_shadows++; if (luma_idx >= 253) @@ -206,9 +335,8 @@ void FrameScope::analyze_video() { if (blue_idx >= 253) clipped_blue++; } - } + } - const double pixel_total = static_cast(width) * static_cast(height); avg_luma = pixel_total > 0.0 ? (luma_sum / pixel_total) : 0.0; } @@ -225,13 +353,8 @@ void FrameScope::analyze_audio() { audio_channels = channels; audio_samples = samples; audio_sample_rate = frame->SampleRate(); - - audio_peak.assign(static_cast(channels), 0.0f); + ensure_audio_buffers(); std::vector rms_sums(static_cast(channels), 0.0); - audio_rms.assign(static_cast(channels), 0.0f); - audio_clipped_samples.assign(static_cast(channels), 0); - audio_waveform_min.assign(static_cast(channels), std::vector(static_cast(audio_buckets), 0.0f)); - audio_waveform_max.assign(static_cast(channels), std::vector(static_cast(audio_buckets), 0.0f)); for (int channel = 0; channel < channels; ++channel) { float* channel_samples = frame->GetAudioSamples(channel); @@ -299,6 +422,10 @@ void FrameScope::rebuild_json() const { video["waveform"]["red"] = json_array_from_vector(waveform_red); video["waveform"]["green"] = json_array_from_vector(waveform_green); video["waveform"]["blue"] = json_array_from_vector(waveform_blue); + + video["vectorscope"] = Json::Value(Json::objectValue); + video["vectorscope"]["size"] = vectorscope_size; + video["vectorscope"]["density"] = json_array_from_vector(vectorscope); } scope_data["video"] = video; @@ -339,6 +466,14 @@ std::string FrameScope::Json() const { return scope_data.toStyledString(); } +std::vector FrameScope::copy_to_int_vector(const std::vector& values) { + std::vector copy(values.size(), 0); + const uint32_t max_int = static_cast(std::numeric_limits::max()); + for (size_t i = 0; i < values.size(); ++i) + copy[i] = static_cast(std::min(values[i], max_int)); + return copy; +} + std::vector FrameScope::GetAudioWaveformMin(int channel) const { if (channel < 0 || channel >= static_cast(audio_waveform_min.size())) return std::vector(); diff --git a/src/FrameScope.h b/src/FrameScope.h index 5676a153..a37a01e9 100644 --- a/src/FrameScope.h +++ b/src/FrameScope.h @@ -16,6 +16,7 @@ #include "Frame.h" #include "Json.h" +#include #include #include #include @@ -39,25 +40,35 @@ namespace openshot { std::shared_ptr frame; int waveform_columns; int audio_buckets; + int vectorscope_size; + bool roi_enabled; + float roi_x; + float roi_y; + float roi_width; + float roi_height; bool video_present; int video_width; int video_height; int waveform_bins; + std::vector waveform_column_map; + int waveform_column_map_width; + int waveform_column_map_columns; double avg_luma; int clipped_shadows; int clipped_highlights; int clipped_red; int clipped_green; int clipped_blue; - std::vector histogram_luma; - std::vector histogram_red; - std::vector histogram_green; - std::vector histogram_blue; - std::vector waveform_luma; - std::vector waveform_red; - std::vector waveform_green; - std::vector waveform_blue; + std::vector histogram_luma; + std::vector histogram_red; + std::vector histogram_green; + std::vector histogram_blue; + std::vector waveform_luma; + std::vector waveform_red; + std::vector waveform_green; + std::vector waveform_blue; + std::vector vectorscope; bool audio_present; int audio_channels; @@ -65,24 +76,30 @@ namespace openshot { int audio_sample_rate; std::vector audio_peak; std::vector audio_rms; - std::vector audio_clipped_samples; + std::vector audio_clipped_samples; std::vector> audio_waveform_min; std::vector> audio_waveform_max; mutable Json::Value scope_data; mutable bool json_dirty; void reset(); + void reset_video(); + void reset_audio(); + void ensure_video_buffers(); + void ensure_audio_buffers(); + void rebuild_waveform_column_map(int width); void analyze_video(); void analyze_audio(); void analyze(); void rebuild_json() const; + static std::vector copy_to_int_vector(const std::vector& values); public: /// Create an empty scope analyzer with default bucket sizes. FrameScope(); /// Construct and immediately analyze a frame. - FrameScope(std::shared_ptr frame, int waveform_columns = 256, int audio_buckets = 256); + FrameScope(std::shared_ptr frame, int waveform_columns = 256, int audio_buckets = 256, int vectorscope_size = 256); /// Replace the current frame and recompute the scope data. void SetFrame(std::shared_ptr new_frame); @@ -96,6 +113,18 @@ namespace openshot { /// Set the number of audio buckets and re-analyze. void SetAudioBuckets(int buckets); + /// Set the vectorscope plane edge length and re-analyze video. + void SetVectorscopeSize(int size); + + /// Set a normalized ROI for video analysis and re-analyze video. + void SetVideoRegionNormalized(float x, float y, float width, float height); + + /// Clear any video ROI and re-analyze the full frame. + void ClearVideoRegion(); + + /// Return whether a video ROI is enabled. + bool HasVideoRegion() const { return roi_enabled; } + /// Return whether the current frame has analyzable video data. bool HasVideo() const { return video_present; } @@ -114,29 +143,62 @@ namespace openshot { /// Return the number of vertical waveform bins. int GetWaveformBins() const { return waveform_bins; } + /// Return the vectorscope plane edge length. + int GetVectorscopeSize() const { return vectorscope_size; } + + /// Return the luma histogram bins by reference. + const std::vector& GetVideoHistogramLumaBins() const { return histogram_luma; } + + /// Return the red histogram bins by reference. + const std::vector& GetVideoHistogramRedBins() const { return histogram_red; } + + /// Return the green histogram bins by reference. + const std::vector& GetVideoHistogramGreenBins() const { return histogram_green; } + + /// Return the blue histogram bins by reference. + const std::vector& GetVideoHistogramBlueBins() const { return histogram_blue; } + + /// Return the flattened luma waveform bins by reference. + const std::vector& GetVideoWaveformLumaBins() const { return waveform_luma; } + + /// Return the flattened red waveform bins by reference. + const std::vector& GetVideoWaveformRedBins() const { return waveform_red; } + + /// Return the flattened green waveform bins by reference. + const std::vector& GetVideoWaveformGreenBins() const { return waveform_green; } + + /// Return the flattened blue waveform bins by reference. + const std::vector& GetVideoWaveformBlueBins() const { return waveform_blue; } + + /// Return the flattened vectorscope density plane by reference. + const std::vector& GetVideoVectorscopeBins() const { return vectorscope; } + + /// Return the flattened vectorscope density plane. + std::vector GetVideoVectorscope() const { return copy_to_int_vector(vectorscope); } + /// Return the luma histogram bins. - std::vector GetVideoHistogramLuma() const { return histogram_luma; } + std::vector GetVideoHistogramLuma() const { return copy_to_int_vector(histogram_luma); } /// Return the red histogram bins. - std::vector GetVideoHistogramRed() const { return histogram_red; } + std::vector GetVideoHistogramRed() const { return copy_to_int_vector(histogram_red); } /// Return the green histogram bins. - std::vector GetVideoHistogramGreen() const { return histogram_green; } + std::vector GetVideoHistogramGreen() const { return copy_to_int_vector(histogram_green); } /// Return the blue histogram bins. - std::vector GetVideoHistogramBlue() const { return histogram_blue; } + std::vector GetVideoHistogramBlue() const { return copy_to_int_vector(histogram_blue); } /// Return the flattened luma waveform bins. - std::vector GetVideoWaveformLuma() const { return waveform_luma; } + std::vector GetVideoWaveformLuma() const { return copy_to_int_vector(waveform_luma); } /// Return the flattened red waveform bins. - std::vector GetVideoWaveformRed() const { return waveform_red; } + std::vector GetVideoWaveformRed() const { return copy_to_int_vector(waveform_red); } /// Return the flattened green waveform bins. - std::vector GetVideoWaveformGreen() const { return waveform_green; } + std::vector GetVideoWaveformGreen() const { return copy_to_int_vector(waveform_green); } /// Return the flattened blue waveform bins. - std::vector GetVideoWaveformBlue() const { return waveform_blue; } + std::vector GetVideoWaveformBlue() const { return copy_to_int_vector(waveform_blue); } /// Return the average luma of the analyzed frame. double GetVideoAverageLuma() const { return avg_luma; } @@ -168,6 +230,15 @@ namespace openshot { /// Return the number of audio waveform buckets. int GetAudioBuckets() const { return audio_buckets; } + /// Return per-channel peak levels. + const std::vector& GetAudioPeakLevelsRef() const { return audio_peak; } + + /// Return per-channel RMS levels. + const std::vector& GetAudioRmsLevelsRef() const { return audio_rms; } + + /// Return per-channel clipped sample counts. + const std::vector& GetAudioClippedSamplesRef() const { return audio_clipped_samples; } + /// Return per-channel peak levels. std::vector GetAudioPeakLevels() const { return audio_peak; } @@ -175,7 +246,7 @@ namespace openshot { std::vector GetAudioRmsLevels() const { return audio_rms; } /// Return per-channel clipped sample counts. - std::vector GetAudioClippedSamples() const { return audio_clipped_samples; } + std::vector GetAudioClippedSamples() const { return copy_to_int_vector(audio_clipped_samples); } /// Return one channel of audio waveform minimum values. std::vector GetAudioWaveformMin(int channel) const; diff --git a/tests/FrameScope.cpp b/tests/FrameScope.cpp index e2d06957..e97c08c0 100644 --- a/tests/FrameScope.cpp +++ b/tests/FrameScope.cpp @@ -75,6 +75,8 @@ TEST_CASE("FrameScope analyzes video histogram and waveform data", "[framescope] CHECK(root["video"]["waveform"]["red"].size() == 4 * 256); CHECK(root["video"]["waveform"]["green"].size() == 4 * 256); CHECK(root["video"]["waveform"]["blue"].size() == 4 * 256); + CHECK(root["video"]["vectorscope"]["size"].asInt() == 256); + CHECK(root["video"]["vectorscope"]["density"].size() == 256 * 256); CHECK(root["video"]["histogram"]["red"][255].asInt() == 1); CHECK(root["video"]["histogram"]["green"][255].asInt() == 1); CHECK(root["video"]["histogram"]["blue"][255].asInt() == 1); @@ -99,9 +101,32 @@ TEST_CASE("FrameScope exposes typed video getters", "[framescope][video][getters CHECK(scope.GetVideoWaveformRed().size() == 4 * 256); CHECK(scope.GetVideoWaveformRed()[1 * 256 + 255] == 1); CHECK(scope.GetVideoWaveformBlue()[3 * 256 + 255] == 1); + CHECK(scope.GetVideoHistogramRedBins()[255] == 1u); + CHECK(scope.GetVideoWaveformGreenBins()[2 * 256 + 255] == 1u); + CHECK(scope.GetVideoVectorscopeBins().size() == 256u * 256u); + CHECK(scope.GetVideoVectorscope().size() == 256 * 256); CHECK(scope.GetVideoAverageLuma() > 0.2); } +TEST_CASE("FrameScope exposes vectorscope bins and configurable size", "[framescope][video][vectorscope]") +{ + FrameScope scope(makeVideoScopeFrame(), 4, 4, 64); + + REQUIRE(scope.HasVideo() == true); + CHECK(scope.GetVectorscopeSize() == 64); + CHECK(scope.GetVideoVectorscopeBins().size() == 64u * 64u); + + uint32_t vectorscope_sum = 0; + for (uint32_t value : scope.GetVideoVectorscopeBins()) + vectorscope_sum += value; + + CHECK(vectorscope_sum == 4u); + + scope.SetVectorscopeSize(32); + CHECK(scope.GetVectorscopeSize() == 32); + CHECK(scope.GetVideoVectorscopeBins().size() == 32u * 32u); +} + TEST_CASE("FrameScope analyzes audio waveform and summary data", "[framescope][audio]") { FrameScope scope(makeAudioScopeFrame(), 4, 4); @@ -134,13 +159,32 @@ TEST_CASE("FrameScope exposes typed audio getters", "[framescope][audio][getters CHECK(scope.GetAudioBuckets() == 4); CHECK(scope.GetAudioPeakLevels().size() == 2); CHECK(scope.GetAudioPeakLevels()[0] == Approx(1.0f)); + CHECK(scope.GetAudioPeakLevelsRef()[1] == Approx(0.8f)); CHECK(scope.GetAudioRmsLevels()[1] > 0.0f); CHECK(scope.GetAudioClippedSamples()[0] == 2); + CHECK(scope.GetAudioClippedSamplesRef()[0] == 2u); CHECK(scope.GetAudioWaveformMin(0).size() == 4); CHECK(scope.GetAudioWaveformMax(1).size() == 4); CHECK(scope.GetAudioWaveformMax(99).empty()); } +TEST_CASE("FrameScope setting changes only affect matching scope families", "[framescope][settings]") +{ + FrameScope scope(makeAudioScopeFrame(), 4, 4, 64); + + REQUIRE(scope.HasAudio() == true); + const std::vector original_peak = scope.GetAudioPeakLevels(); + scope.SetVectorscopeSize(32); + CHECK(scope.GetAudioPeakLevels() == original_peak); + + auto video_frame = makeVideoScopeFrame(); + scope.SetFrame(video_frame); + REQUIRE(scope.HasVideo() == true); + const std::vector original_vectorscope = scope.GetVideoVectorscopeBins(); + scope.SetAudioBuckets(8); + CHECK(scope.GetVideoVectorscopeBins() == original_vectorscope); +} + TEST_CASE("FrameScope Json string parses cleanly", "[framescope][json]") { FrameScope scope(makeAudioScopeFrame(), 8, 4);